Suppot RTMPOSE classes
This commit is contained in:
+3
-1
@@ -50,7 +50,9 @@ enum ModelType {
|
|||||||
ONNXSAM3 = 28,
|
ONNXSAM3 = 28,
|
||||||
RTSAM3 = 29,
|
RTSAM3 = 29,
|
||||||
ONNXYOLO = 30,
|
ONNXYOLO = 30,
|
||||||
RTYOLO = 31
|
RTYOLO = 31,
|
||||||
|
ONNXRTMPOSE=32,
|
||||||
|
RTRTMPOSE=33
|
||||||
};
|
};
|
||||||
enum TrackerType {
|
enum TrackerType {
|
||||||
BYTETRACK = 0,
|
BYTETRACK = 0,
|
||||||
|
|||||||
@@ -241,7 +241,9 @@ namespace ANSCENTER
|
|||||||
ONNXSAM3=28,
|
ONNXSAM3=28,
|
||||||
RTSAM3 = 29,
|
RTSAM3 = 29,
|
||||||
ONNXYOLO = 30,
|
ONNXYOLO = 30,
|
||||||
RTYOLO = 31
|
RTYOLO = 31,
|
||||||
|
ONNXRTMPOSE=32,
|
||||||
|
RTRTMPOSE=33
|
||||||
};
|
};
|
||||||
enum TrackerType {
|
enum TrackerType {
|
||||||
BYTETRACK = 0,
|
BYTETRACK = 0,
|
||||||
|
|||||||
@@ -0,0 +1,673 @@
|
|||||||
|
#include "ANSONNXRTMPOSE.h"
|
||||||
|
#include "Utility.h"
|
||||||
|
#include "ANSLicense.h" // ANS_DBG macro for DebugView
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cmath>
|
||||||
|
#include <chrono>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
namespace ANSCENTER {
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ONNXRtmPose — BasicOrtHandler subclass
|
||||||
|
// ====================================================================
|
||||||
|
ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, unsigned int _num_threads)
|
||||||
|
: BasicOrtHandler(_onnx_path, _num_threads)
|
||||||
|
{
|
||||||
|
resolveInputShape();
|
||||||
|
}
|
||||||
|
|
||||||
|
ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, EngineType engineType,
|
||||||
|
unsigned int _num_threads)
|
||||||
|
: BasicOrtHandler(_onnx_path, engineType, _num_threads)
|
||||||
|
{
|
||||||
|
resolveInputShape();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// resolveInputShape — read input/output dims off the model.
|
||||||
|
// Input : 1x3xHxW (RTMPose body-2d default 1x3x256x192)
|
||||||
|
// Output : simcc_x [1, K, ext_w], simcc_y [1, K, ext_h]
|
||||||
|
// The keypoint count K is read off the first output's dim[1].
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
void ONNXRtmPose::resolveInputShape() {
|
||||||
|
if (input_node_dims.size() >= 4) {
|
||||||
|
const int h = static_cast<int>(input_node_dims[2]);
|
||||||
|
const int w = static_cast<int>(input_node_dims[3]);
|
||||||
|
if (h > 0) m_input_h = h;
|
||||||
|
if (w > 0) m_input_w = w;
|
||||||
|
}
|
||||||
|
if (!output_node_dims.empty() && output_node_dims[0].size() >= 2) {
|
||||||
|
const int k = static_cast<int>(output_node_dims[0][1]);
|
||||||
|
if (k > 0) m_num_points = k;
|
||||||
|
}
|
||||||
|
ANS_DBG("ONNXRtmPose", "resolveInputShape: %dx%d kps=%d", m_input_w, m_input_h, m_num_points);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// GetAffineTransform — direct port of utils.h.
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
cv::Mat ONNXRtmPose::GetAffineTransform(float center_x, float center_y,
|
||||||
|
float scale_w, float scale_h,
|
||||||
|
int output_w, int output_h, bool inverse)
|
||||||
|
{
|
||||||
|
cv::Point2f src1(center_x, center_y);
|
||||||
|
cv::Point2f src2(center_x, center_y - scale_w * 0.5f);
|
||||||
|
cv::Point2f src3(src2.x - (src1.y - src2.y),
|
||||||
|
src2.y + (src1.x - src2.x));
|
||||||
|
|
||||||
|
const float dcx = output_w * 0.5f;
|
||||||
|
const float dcy = output_h * 0.5f;
|
||||||
|
cv::Point2f dst1(dcx, dcy);
|
||||||
|
cv::Point2f dst2(dcx, dcy - output_w * 0.5f);
|
||||||
|
cv::Point2f dst3(dst2.x - (dst1.y - dst2.y),
|
||||||
|
dst2.y + (dst1.x - dst2.x));
|
||||||
|
|
||||||
|
cv::Point2f srcPts[3] = { src1, src2, src3 };
|
||||||
|
cv::Point2f dstPts[3] = { dst1, dst2, dst3 };
|
||||||
|
return inverse ? cv::getAffineTransform(dstPts, srcPts)
|
||||||
|
: cv::getAffineTransform(srcPts, dstPts);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// CropImageByDetectBox — aspect-fit + 1.2x context expansion.
|
||||||
|
// Direct port of rtmpose_onnxruntime.cpp::CropImageByDetectBox.
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
void ONNXRtmPose::CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box,
|
||||||
|
cv::Mat& crop, cv::Mat& affine_rev)
|
||||||
|
{
|
||||||
|
float w = static_cast<float>(box.width);
|
||||||
|
float h = static_cast<float>(box.height);
|
||||||
|
const float cx = box.x + w * 0.5f;
|
||||||
|
const float cy = box.y + h * 0.5f;
|
||||||
|
|
||||||
|
const float aspect = static_cast<float>(m_input_w) / static_cast<float>(m_input_h);
|
||||||
|
if (w > aspect * h) h = w / aspect;
|
||||||
|
else w = h * aspect;
|
||||||
|
const float scale_w = w * 1.2f;
|
||||||
|
const float scale_h = h * 1.2f;
|
||||||
|
|
||||||
|
cv::Mat affine = GetAffineTransform(cx, cy, scale_w, scale_h,
|
||||||
|
m_input_w, m_input_h, false);
|
||||||
|
affine_rev = GetAffineTransform(cx, cy, scale_w, scale_h,
|
||||||
|
m_input_w, m_input_h, true);
|
||||||
|
|
||||||
|
cv::warpAffine(image, crop, affine,
|
||||||
|
cv::Size(m_input_w, m_input_h), cv::INTER_LINEAR);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// transformInto — input is a 192x256 BGR person crop. Convert to
|
||||||
|
// RGB, ImageNet-normalize, lay out CHW into outBuffer, build the
|
||||||
|
// Ort::Value view over outBuffer.data(). Mirrors the design of
|
||||||
|
// ONNXYOLO::transformInto: per-call backing storage is owned by the
|
||||||
|
// caller so two threads sharing one ONNXRtmPose can preprocess and
|
||||||
|
// Run() concurrently without racing on input_values_handler.
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
Ort::Value ONNXRtmPose::transformInto(const cv::Mat& cropBGR,
|
||||||
|
std::vector<float>& outBuffer,
|
||||||
|
std::vector<int64_t>& outDims)
|
||||||
|
{
|
||||||
|
cv::Mat rgb;
|
||||||
|
cv::cvtColor(cropBGR, rgb, cv::COLOR_BGR2RGB);
|
||||||
|
|
||||||
|
cv::Mat floatImg;
|
||||||
|
rgb.convertTo(floatImg, CV_32FC3);
|
||||||
|
|
||||||
|
const int H = floatImg.rows;
|
||||||
|
const int W = floatImg.cols;
|
||||||
|
const size_t HW = static_cast<size_t>(H) * W;
|
||||||
|
|
||||||
|
outDims = { 1, 3, static_cast<int64_t>(H), static_cast<int64_t>(W) };
|
||||||
|
outBuffer.resize(3 * HW);
|
||||||
|
float* data = outBuffer.data();
|
||||||
|
|
||||||
|
// HWC -> CHW + (pixel - mean) / std. RGB channel order.
|
||||||
|
for (int y = 0; y < H; ++y) {
|
||||||
|
const float* row = floatImg.ptr<float>(y);
|
||||||
|
for (int x = 0; x < W; ++x) {
|
||||||
|
const float r = row[x * 3 + 0];
|
||||||
|
const float g = row[x * 3 + 1];
|
||||||
|
const float b = row[x * 3 + 2];
|
||||||
|
data[0 * HW + y * W + x] = (r - MEAN_R) / STD_R;
|
||||||
|
data[1 * HW + y * W + x] = (g - MEAN_G) / STD_G;
|
||||||
|
data[2 * HW + y * W + x] = (b - MEAN_B) / STD_B;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ort::Value::CreateTensor<float>(
|
||||||
|
*memory_info_handler,
|
||||||
|
outBuffer.data(),
|
||||||
|
outBuffer.size(),
|
||||||
|
outDims.data(),
|
||||||
|
outDims.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
// BasicOrtHandler pure-virtual override. Thin shim that writes into
|
||||||
|
// the shared base members (legacy single-threaded caller path).
|
||||||
|
Ort::Value ONNXRtmPose::transform(const cv::Mat& mat)
|
||||||
|
{
|
||||||
|
Ort::Value v = transformInto(mat, input_values_handler, input_node_dims);
|
||||||
|
input_tensor_size = input_values_handler.size();
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
// RTMPose is single-person per call — batched inference doesn't apply.
|
||||||
|
Ort::Value ONNXRtmPose::transformBatch(const std::vector<cv::Mat>& /*images*/)
|
||||||
|
{
|
||||||
|
throw std::runtime_error("ONNXRtmPose::transformBatch: not supported (single-person model)");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// estimate — full per-bbox pipeline.
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
Object ONNXRtmPose::estimate(const cv::Mat& image, const cv::Rect& box)
|
||||||
|
{
|
||||||
|
Object obj{};
|
||||||
|
if (image.empty() || box.width <= 0 || box.height <= 0) {
|
||||||
|
ANS_DBG("ONNXRtmPose", "estimate: invalid input empty=%d box=%dx%d",
|
||||||
|
image.empty() ? 1 : 0, box.width, box.height);
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Affine crop -> 192x256 BGR person image, plus inverse-affine
|
||||||
|
// to map output coords back to original image.
|
||||||
|
cv::Mat crop, affine_reverse;
|
||||||
|
CropImageByDetectBox(image, box, crop, affine_reverse);
|
||||||
|
if (crop.empty()) {
|
||||||
|
ANS_DBG("ONNXRtmPose", "estimate: warpAffine produced empty crop");
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Per-call backing storage so multiple threads sharing one
|
||||||
|
// ONNXRtmPose can Run() in parallel — see comment on
|
||||||
|
// transformInto.
|
||||||
|
std::vector<float> inputBuffer;
|
||||||
|
std::vector<int64_t> inputDims;
|
||||||
|
Ort::Value inputTensor = transformInto(crop, inputBuffer, inputDims);
|
||||||
|
|
||||||
|
auto outputTensors = ort_session->Run(
|
||||||
|
Ort::RunOptions{ nullptr },
|
||||||
|
input_node_names.data(),
|
||||||
|
&inputTensor, 1,
|
||||||
|
output_node_names.data(),
|
||||||
|
num_outputs);
|
||||||
|
|
||||||
|
if (outputTensors.size() < 2) {
|
||||||
|
ANS_DBG("ONNXRtmPose", "estimate: expected 2 outputs (simcc_x,simcc_y), got %zu",
|
||||||
|
outputTensors.size());
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode SimCC. Disambiguate which output is X vs Y by length
|
||||||
|
// (X output's last dim is proportional to input W; Y's to H).
|
||||||
|
float* out0 = outputTensors[0].GetTensorMutableData<float>();
|
||||||
|
float* out1 = outputTensors[1].GetTensorMutableData<float>();
|
||||||
|
auto shape0 = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||||
|
auto shape1 = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();
|
||||||
|
if (shape0.size() < 3 || shape1.size() < 3 ||
|
||||||
|
shape0[1] != shape1[1] || shape0[1] <= 0) {
|
||||||
|
ANS_DBG("ONNXRtmPose", "estimate: bad output shapes (rank or K mismatch)");
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
const int joint_num = static_cast<int>(shape0[1]);
|
||||||
|
int W0 = static_cast<int>(shape0[2]);
|
||||||
|
int H1 = static_cast<int>(shape1[2]);
|
||||||
|
float* simcc_x = out0;
|
||||||
|
float* simcc_y = out1;
|
||||||
|
int extW = W0;
|
||||||
|
int extH = H1;
|
||||||
|
// If the first output looks H-shaped (closer to 2*input_h than
|
||||||
|
// 2*input_w), swap so simcc_x always indexes the W-axis.
|
||||||
|
const float ratioFirstW = (m_input_w > 0) ? float(W0) / float(m_input_w) : 0.f;
|
||||||
|
const float ratioFirstH = (m_input_h > 0) ? float(W0) / float(m_input_h) : 0.f;
|
||||||
|
if (std::abs(ratioFirstW - 2.f) > std::abs(ratioFirstH - 2.f)) {
|
||||||
|
std::swap(simcc_x, simcc_y);
|
||||||
|
extW = H1;
|
||||||
|
extH = W0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const double m0 = affine_reverse.at<double>(0, 0);
|
||||||
|
const double m1 = affine_reverse.at<double>(0, 1);
|
||||||
|
const double m2 = affine_reverse.at<double>(0, 2);
|
||||||
|
const double m3 = affine_reverse.at<double>(1, 0);
|
||||||
|
const double m4 = affine_reverse.at<double>(1, 1);
|
||||||
|
const double m5 = affine_reverse.at<double>(1, 2);
|
||||||
|
|
||||||
|
const float imgW = static_cast<float>(image.cols);
|
||||||
|
const float imgH = static_cast<float>(image.rows);
|
||||||
|
|
||||||
|
std::vector<float> kps;
|
||||||
|
kps.reserve(static_cast<size_t>(joint_num) * 3);
|
||||||
|
std::vector<cv::Point2f> polygon;
|
||||||
|
polygon.reserve(joint_num);
|
||||||
|
std::stringstream xss, yss;
|
||||||
|
float meanScore = 0.f;
|
||||||
|
int meanCount = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < joint_num; ++i) {
|
||||||
|
const float* x_ptr = simcc_x + static_cast<size_t>(i) * extW;
|
||||||
|
const float* y_ptr = simcc_y + static_cast<size_t>(i) * extH;
|
||||||
|
const int px = static_cast<int>(std::max_element(x_ptr, x_ptr + extW) - x_ptr);
|
||||||
|
const int py = static_cast<int>(std::max_element(y_ptr, y_ptr + extH) - y_ptr);
|
||||||
|
const float score = std::max(x_ptr[px], y_ptr[py]);
|
||||||
|
|
||||||
|
// SimCC oversampling factor 2 — same constant as the reference impl.
|
||||||
|
const double raw_x = px * 0.5;
|
||||||
|
const double raw_y = py * 0.5;
|
||||||
|
|
||||||
|
float wx = static_cast<float>(m0 * raw_x + m1 * raw_y + m2);
|
||||||
|
float wy = static_cast<float>(m3 * raw_x + m4 * raw_y + m5);
|
||||||
|
wx = std::clamp(wx, 0.f, imgW - 1.f);
|
||||||
|
wy = std::clamp(wy, 0.f, imgH - 1.f);
|
||||||
|
|
||||||
|
kps.push_back(wx);
|
||||||
|
kps.push_back(wy);
|
||||||
|
kps.push_back(score);
|
||||||
|
polygon.emplace_back(wx, wy);
|
||||||
|
|
||||||
|
xss << wx;
|
||||||
|
yss << wy;
|
||||||
|
if (i + 1 < joint_num) { xss << ";"; yss << ";"; }
|
||||||
|
|
||||||
|
if (score > 0.f) { meanScore += score; ++meanCount; }
|
||||||
|
}
|
||||||
|
|
||||||
|
obj.classId = 0;
|
||||||
|
obj.confidence = (meanCount > 0) ? meanScore / meanCount : 0.f;
|
||||||
|
obj.box = box;
|
||||||
|
obj.kps = std::move(kps);
|
||||||
|
obj.polygon = std::move(polygon);
|
||||||
|
obj.extraInfo = xss.str() + "|" + yss.str();
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
ANS_DBG("ONNXRtmPose", "estimate: EXCEPTION '%s'", e.what());
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSONNXRTMPOSE — ANSODBase wrapper
|
||||||
|
// ====================================================================
|
||||||
|
ANSONNXRTMPOSE::~ANSONNXRTMPOSE() {
|
||||||
|
try { Destroy(); }
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::~ANSONNXRTMPOSE", e.what(), __FILE__, __LINE__);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::Destroy() {
|
||||||
|
try {
|
||||||
|
m_ortEngine.reset();
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::Destroy", e.what(), __FILE__, __LINE__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
|
||||||
|
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false;
|
||||||
|
// ONNX models don't need an explicit precision-conversion pass here;
|
||||||
|
// the EP picks FP16 / FP32 at session-creation time. Mirrors
|
||||||
|
// ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder
|
||||||
|
// resolution.
|
||||||
|
optimizedModelFolder = _modelFolder;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::InitOrtEngine() {
|
||||||
|
try {
|
||||||
|
if (!FileExist(_modelFilePath)) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::InitOrtEngine",
|
||||||
|
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: missing onnx '%s'", _modelFilePath.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
m_ortEngine = std::make_shared<ONNXRtmPose>(_modelFilePath);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: ready %dx%d kps=%d",
|
||||||
|
m_ortEngine->inputWidth(), m_ortEngine->inputHeight(),
|
||||||
|
m_ortEngine->numKeypoints());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::InitOrtEngine(ANSCENTER::EngineType engineType) {
|
||||||
|
try {
|
||||||
|
if (!FileExist(_modelFilePath)) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::InitOrtEngine",
|
||||||
|
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
m_ortEngine = std::make_shared<ONNXRtmPose>(_modelFilePath, engineType);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine(EP=%d): ready %dx%d kps=%d",
|
||||||
|
static_cast<int>(engineType),
|
||||||
|
m_ortEngine->inputWidth(), m_ortEngine->inputHeight(),
|
||||||
|
m_ortEngine->numKeypoints());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// WarmUpEngine — single dummy inference to amortise first-call cost
|
||||||
|
// (CUDA arena alloc / cuDNN algo selection / DML shader compile).
|
||||||
|
// Non-fatal on failure: the engine is fully loaded before this runs.
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
void ANSONNXRTMPOSE::WarmUpEngine() {
|
||||||
|
if (!m_ortEngine) return;
|
||||||
|
const auto et = m_ortEngine->getEngineType();
|
||||||
|
if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) {
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "Warm-up skipped (CPU/OpenVINO EP)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const int W = m_ortEngine->inputWidth();
|
||||||
|
const int H = m_ortEngine->inputHeight();
|
||||||
|
cv::Mat dummy(H, W, CV_8UC3, cv::Scalar(128, 128, 128));
|
||||||
|
const cv::Rect wholeImage(0, 0, W, H);
|
||||||
|
|
||||||
|
auto t0 = std::chrono::steady_clock::now();
|
||||||
|
(void)m_ortEngine->estimate(dummy, wholeImage);
|
||||||
|
auto t1 = std::chrono::steady_clock::now();
|
||||||
|
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "Warm-up done: %lld ms (%dx%d)", (long long)ms, W, H);
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine",
|
||||||
|
std::string("Warm-up skipped (non-fatal): ") + e.what(),
|
||||||
|
__FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "Warm-up: EXCEPTION '%s'", e.what());
|
||||||
|
}
|
||||||
|
catch (...) {
|
||||||
|
_logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine",
|
||||||
|
"Warm-up skipped (unknown exception)", __FILE__, __LINE__);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword,
|
||||||
|
std::string& labelMap)
|
||||||
|
{
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ModelLoadingGuard mlg(_modelLoading);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "Initialize: zip='%s'", modelZipFilePath.c_str());
|
||||||
|
try {
|
||||||
|
_modelLoadValid = false;
|
||||||
|
bool result = ANSODBase::Initialize(licenseKey, modelConfig,
|
||||||
|
modelZipFilePath, modelZipPassword, labelMap);
|
||||||
|
if (!result) return false;
|
||||||
|
|
||||||
|
_modelConfig = modelConfig;
|
||||||
|
_modelConfig.modelType = ModelType::ONNXRTMPOSE;
|
||||||
|
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256;
|
||||||
|
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192;
|
||||||
|
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
|
||||||
|
_modelConfig.numKPS = 26;
|
||||||
|
if (_modelConfig.kpsThreshold <= 0)
|
||||||
|
_modelConfig.kpsThreshold = KPS_THRESHOLD;
|
||||||
|
KPS_THRESHOLD = _modelConfig.kpsThreshold;
|
||||||
|
|
||||||
|
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||||
|
if (FileExist(_modelConfigFile)) {
|
||||||
|
ModelType modelType;
|
||||||
|
std::vector<int> inputShape;
|
||||||
|
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||||
|
if (inputShape.size() == 2) {
|
||||||
|
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||||
|
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||||
|
std::ifstream isValid(_classFilePath);
|
||||||
|
if (!isValid) LoadClassesFromString();
|
||||||
|
else LoadClassesFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
labelMap.clear();
|
||||||
|
if (!_classes.empty())
|
||||||
|
labelMap = VectorToCommaSeparatedString(_classes);
|
||||||
|
|
||||||
|
if (this->_loadEngineOnCreation) {
|
||||||
|
if (!InitOrtEngine()) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::Initialize",
|
||||||
|
"Failed to create ONNX Runtime engine: " + _modelFilePath,
|
||||||
|
__FILE__, __LINE__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// Sync ModelConfig.numKPS to whatever the model actually has,
|
||||||
|
// so consumers reading GetModelConfig() see the truth.
|
||||||
|
if (m_ortEngine && m_ortEngine->numKeypoints() > 0)
|
||||||
|
_modelConfig.numKPS = m_ortEngine->numKeypoints();
|
||||||
|
WarmUpEngine();
|
||||||
|
}
|
||||||
|
|
||||||
|
_modelLoadValid = true;
|
||||||
|
_isInitialized = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::Initialize", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "Initialize: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::LoadModel(const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword)
|
||||||
|
{
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ModelLoadingGuard mlg(_modelLoading);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "LoadModel: zip='%s'", modelZipFilePath.c_str());
|
||||||
|
try {
|
||||||
|
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
|
||||||
|
if (!result) return false;
|
||||||
|
|
||||||
|
_modelConfig.modelType = ModelType::ONNXRTMPOSE;
|
||||||
|
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256;
|
||||||
|
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192;
|
||||||
|
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
|
||||||
|
_modelConfig.numKPS = 26;
|
||||||
|
if (_modelConfig.kpsThreshold <= 0)
|
||||||
|
_modelConfig.kpsThreshold = KPS_THRESHOLD;
|
||||||
|
KPS_THRESHOLD = _modelConfig.kpsThreshold;
|
||||||
|
|
||||||
|
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||||
|
if (FileExist(_modelConfigFile)) {
|
||||||
|
ModelType modelType;
|
||||||
|
std::vector<int> inputShape;
|
||||||
|
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||||
|
if (inputShape.size() == 2) {
|
||||||
|
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||||
|
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||||
|
std::ifstream isValid(_classFilePath);
|
||||||
|
if (!isValid) LoadClassesFromString();
|
||||||
|
else LoadClassesFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this->_loadEngineOnCreation) {
|
||||||
|
if (!InitOrtEngine()) { _modelLoadValid = false; return false; }
|
||||||
|
if (m_ortEngine && m_ortEngine->numKeypoints() > 0)
|
||||||
|
_modelConfig.numKPS = m_ortEngine->numKeypoints();
|
||||||
|
WarmUpEngine();
|
||||||
|
}
|
||||||
|
_modelLoadValid = true;
|
||||||
|
_isInitialized = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::LoadModel", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "LoadModel: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSONNXRTMPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
std::string modelName, std::string className,
|
||||||
|
const std::string& modelFolder,
|
||||||
|
std::string& labelMap)
|
||||||
|
{
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ModelLoadingGuard mlg(_modelLoading);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: folder='%s' model='%s'",
|
||||||
|
modelFolder.c_str(), modelName.c_str());
|
||||||
|
try {
|
||||||
|
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig,
|
||||||
|
modelName, className,
|
||||||
|
modelFolder, labelMap);
|
||||||
|
if (!result) return false;
|
||||||
|
|
||||||
|
_modelConfig = modelConfig;
|
||||||
|
_modelConfig.modelType = ModelType::ONNXRTMPOSE;
|
||||||
|
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256;
|
||||||
|
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192;
|
||||||
|
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
|
||||||
|
_modelConfig.numKPS = 26;
|
||||||
|
if (_modelConfig.kpsThreshold <= 0)
|
||||||
|
_modelConfig.kpsThreshold = KPS_THRESHOLD;
|
||||||
|
KPS_THRESHOLD = _modelConfig.kpsThreshold;
|
||||||
|
|
||||||
|
std::string _modelName = modelName;
|
||||||
|
if (_modelName.empty()) _modelName = "train_last";
|
||||||
|
std::string modelFullName = _modelName + ".onnx";
|
||||||
|
|
||||||
|
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
|
||||||
|
if (FileExist(_modelConfigFile)) {
|
||||||
|
ModelType modelType;
|
||||||
|
std::vector<int> inputShape;
|
||||||
|
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||||
|
if (inputShape.size() == 2) {
|
||||||
|
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||||
|
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
_classFilePath = CreateFilePath(_modelFolder, className);
|
||||||
|
std::ifstream isValid(_classFilePath);
|
||||||
|
if (!isValid) LoadClassesFromString();
|
||||||
|
else LoadClassesFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
labelMap.clear();
|
||||||
|
if (!_classes.empty())
|
||||||
|
labelMap = VectorToCommaSeparatedString(_classes);
|
||||||
|
|
||||||
|
if (this->_loadEngineOnCreation) {
|
||||||
|
if (!InitOrtEngine()) { _modelLoadValid = false; return false; }
|
||||||
|
if (m_ortEngine && m_ortEngine->numKeypoints() > 0)
|
||||||
|
_modelConfig.numKPS = m_ortEngine->numKeypoints();
|
||||||
|
WarmUpEngine();
|
||||||
|
}
|
||||||
|
_modelLoadValid = true;
|
||||||
|
_isInitialized = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<Object> ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR) {
|
||||||
|
return RunInference(inputImgBGR, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<Object> ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR,
|
||||||
|
const std::string& camera_id)
|
||||||
|
{
|
||||||
|
if (_modelLoading.load()) return {};
|
||||||
|
{
|
||||||
|
auto lock = TryLockWithTimeout("ANSONNXRTMPOSE::RunInference");
|
||||||
|
if (!lock.owns_lock()) return {};
|
||||||
|
if (!_modelLoadValid) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not loaded", __FILE__, __LINE__);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (!_licenseValid) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::RunInference", "Invalid license", __FILE__, __LINE__);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (!_isInitialized) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not initialized", __FILE__, __LINE__);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10)
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
try { return DetectObjects(inputImgBGR, camera_id); }
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::RunInference", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "RunInference: EXCEPTION '%s'", e.what());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
// DetectObjects — top-down: with no external bbox, treat the whole
|
||||||
|
// input image as the person crop and run RTMPose. Higher-level
|
||||||
|
// orchestration that already has per-person bboxes should call
|
||||||
|
// ANSODBase::RunInference(input, vector<Rect>, cam) — that overload
|
||||||
|
// crops each bbox and routes back here per crop.
|
||||||
|
// ------------------------------------------------------------------
|
||||||
|
std::vector<Object> ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage,
|
||||||
|
const std::string& camera_id)
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
// Snapshot engine handle under brief lock; dereference outside
|
||||||
|
// (mirrors ANSONNXYOLO::DetectObjects so concurrent inference
|
||||||
|
// threads don't serialise on _mutex for the full ORT Run()).
|
||||||
|
std::shared_ptr<ONNXRtmPose> engine;
|
||||||
|
std::string firstClass;
|
||||||
|
{
|
||||||
|
auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects");
|
||||||
|
if (!lk.owns_lock()) return {};
|
||||||
|
if (!m_ortEngine) {
|
||||||
|
_logger.LogError("ANSONNXRTMPOSE::DetectObjects",
|
||||||
|
"ORT engine is null", __FILE__, __LINE__);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
engine = m_ortEngine; // ref-count++
|
||||||
|
if (!_classes.empty()) firstClass = _classes.front();
|
||||||
|
}
|
||||||
|
|
||||||
|
const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows);
|
||||||
|
Object obj = engine->estimate(inputImage, wholeImage);
|
||||||
|
if (obj.kps.empty()) return {};
|
||||||
|
|
||||||
|
obj.cameraId = camera_id;
|
||||||
|
obj.className = firstClass.empty() ? std::string("person") : firstClass;
|
||||||
|
|
||||||
|
std::vector<Object> out;
|
||||||
|
out.push_back(std::move(obj));
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: kps=%d cam='%s'",
|
||||||
|
engine->numKeypoints(), camera_id.c_str());
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSONNXRTMPOSE::DetectObjects", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: EXCEPTION '%s'", e.what());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace ANSCENTER
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
#ifndef ANSONNXRTMPOSE_H
|
||||||
|
#define ANSONNXRTMPOSE_H
|
||||||
|
#pragma once
|
||||||
|
#include "ANSEngineCommon.h"
|
||||||
|
#include "ONNXEngine.h"
|
||||||
|
|
||||||
|
namespace ANSCENTER {
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ONNXRtmPose — BasicOrtHandler subclass for RTMPose (SimCC) ONNX
|
||||||
|
//
|
||||||
|
// Ported from C:\Projects\AIVISION\RTM_POSE\OnnxRuntime-CPU\
|
||||||
|
// OnnxRunTime-CPU\rtmpose_onnxruntime.cpp but using BasicOrtHandler
|
||||||
|
// so the session benefits from the shared EP loader (CUDA / TensorRT
|
||||||
|
// / DirectML / OpenVINO / CPU auto-detect), per-session high-perf
|
||||||
|
// options, EP fallback recovery, and the wstring path handling that
|
||||||
|
// BasicOrtHandler centralises.
|
||||||
|
//
|
||||||
|
// RTMPose is a top-down single-person pose estimator:
|
||||||
|
// Input : person crop, 192x256 RGB, ImageNet-normalized
|
||||||
|
// Output : two SimCC heatmaps
|
||||||
|
// simcc_x [1, K, ext_w]
|
||||||
|
// simcc_y [1, K, ext_h]
|
||||||
|
// K = number of keypoints (e.g. 17 / 26 / 133)
|
||||||
|
// ext_w / ext_h = SimCC oversampled dims (typ. 2 * input)
|
||||||
|
//
|
||||||
|
// Class name is mixed-case (not ONNXRTMPOSE) to avoid a collision with
|
||||||
|
// the unscoped enum value ModelType::ONNXRTMPOSE in ANSLicense.h —
|
||||||
|
// identical spelling would shadow the type.
|
||||||
|
// ====================================================================
|
||||||
|
class ONNXENGINE_API ONNXRtmPose : public BasicOrtHandler
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
explicit ONNXRtmPose(const std::string& _onnx_path,
|
||||||
|
unsigned int _num_threads = 1);
|
||||||
|
explicit ONNXRtmPose(const std::string& _onnx_path,
|
||||||
|
EngineType engineType,
|
||||||
|
unsigned int _num_threads = 1);
|
||||||
|
~ONNXRtmPose() override = default;
|
||||||
|
|
||||||
|
// Full per-bbox pipeline. Returns one Object with kps in original-
|
||||||
|
// image coords (kps layout: [x, y, score] * K). Empty kps means
|
||||||
|
// the call short-circuited (invalid box, runtime error, etc.).
|
||||||
|
Object estimate(const cv::Mat& image, const cv::Rect& box);
|
||||||
|
|
||||||
|
// Read-back accessors so the wrapper can introspect once the
|
||||||
|
// session is alive (used to populate _modelConfig.numKPS, log a
|
||||||
|
// sanity line, etc.).
|
||||||
|
int inputWidth() const { return m_input_w; }
|
||||||
|
int inputHeight() const { return m_input_h; }
|
||||||
|
int numKeypoints() const { return m_num_points; }
|
||||||
|
|
||||||
|
private:
|
||||||
|
// Override the BasicOrtHandler pure virtuals. transform() takes a
|
||||||
|
// pre-cropped 192x256 BGR person image and produces the normalized
|
||||||
|
// CHW tensor. transformBatch() throws — RTMPose is single-person.
|
||||||
|
Ort::Value transform(const cv::Mat& mat) override;
|
||||||
|
Ort::Value transformBatch(const std::vector<cv::Mat>& images) override;
|
||||||
|
|
||||||
|
// Per-call preprocess that writes into a caller-provided buffer +
|
||||||
|
// dims vector instead of the BasicOrtHandler members (mirrors
|
||||||
|
// ONNXYOLO::transformInto). Lets concurrent threads sharing one
|
||||||
|
// ONNXRtmPose instance preprocess and infer in parallel without
|
||||||
|
// racing on input_values_handler.
|
||||||
|
Ort::Value transformInto(const cv::Mat& cropBGR,
|
||||||
|
std::vector<float>& outBuffer,
|
||||||
|
std::vector<int64_t>& outDims);
|
||||||
|
|
||||||
|
// Affine helpers — direct port of utils.h::GetAffineTransform
|
||||||
|
// and rtmpose_onnxruntime.cpp::CropImageByDetectBox.
|
||||||
|
static cv::Mat GetAffineTransform(float center_x, float center_y,
|
||||||
|
float scale_w, float scale_h,
|
||||||
|
int output_w, int output_h,
|
||||||
|
bool inverse = false);
|
||||||
|
void CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box,
|
||||||
|
cv::Mat& crop, cv::Mat& affine_rev);
|
||||||
|
|
||||||
|
// Read input geometry off the model (1x3xHxW). Falls back to
|
||||||
|
// 192x256 (the canonical RTMPose body-2d shape) if the ONNX
|
||||||
|
// export uses dynamic dims.
|
||||||
|
void resolveInputShape();
|
||||||
|
|
||||||
|
// Fixed RTMPose model geometry (read off the model at construction;
|
||||||
|
// defaults match the reference impl).
|
||||||
|
int m_input_w{ 192 };
|
||||||
|
int m_input_h{ 256 };
|
||||||
|
int m_num_points{ 26 };
|
||||||
|
|
||||||
|
// ImageNet normalization. Matches the IMAGE_MEAN / IMAGE_STD
|
||||||
|
// constants in C:\Projects\AIVISION\RTM_POSE\OnnxRuntime-CPU\
|
||||||
|
// OnnxRunTime-CPU\rtmpose_utils.h.
|
||||||
|
const float MEAN_R{ 123.675f };
|
||||||
|
const float MEAN_G{ 116.28f };
|
||||||
|
const float MEAN_B{ 103.53f };
|
||||||
|
const float STD_R { 58.395f };
|
||||||
|
const float STD_G { 57.12f };
|
||||||
|
const float STD_B { 57.375f };
|
||||||
|
};
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSONNXRTMPOSE — ANSODBase wrapper around ONNXRtmPose
|
||||||
|
//
|
||||||
|
// Mirrors the structure of ANSONNXYOLO. Owns a shared_ptr<ONNXRtmPose>
|
||||||
|
// so a concurrent inference thread can copy the handle under a brief
|
||||||
|
// lock and dereference it lock-free, which lets multiple AI tasks
|
||||||
|
// share one ANSONNXRTMPOSE without serialising on the engine mutex.
|
||||||
|
// ====================================================================
|
||||||
|
class ANSENGINE_API ANSONNXRTMPOSE : public ANSODBase {
|
||||||
|
public:
|
||||||
|
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword,
|
||||||
|
std::string& labelMap) override;
|
||||||
|
|
||||||
|
bool LoadModel(const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword) override;
|
||||||
|
|
||||||
|
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
std::string modelName, std::string className,
|
||||||
|
const std::string& modelFolder,
|
||||||
|
std::string& labelMap) override;
|
||||||
|
|
||||||
|
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override;
|
||||||
|
|
||||||
|
std::vector<Object> RunInference(const cv::Mat& input) override;
|
||||||
|
std::vector<Object> RunInference(const cv::Mat& input,
|
||||||
|
const std::string& camera_id) override;
|
||||||
|
|
||||||
|
bool Destroy() override;
|
||||||
|
~ANSONNXRTMPOSE();
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::string _modelFilePath;
|
||||||
|
bool _modelLoadValid{ false };
|
||||||
|
|
||||||
|
// Keypoint confidence floor. The SimCC max score is emitted as
|
||||||
|
// kps[2] regardless; this is just the threshold used to compute
|
||||||
|
// the per-Object confidence average.
|
||||||
|
float KPS_THRESHOLD{ 0.3f };
|
||||||
|
|
||||||
|
// ORT engine. shared_ptr (mirrors ANSONNXYOLO) so an inference
|
||||||
|
// thread holding a local copy keeps the underlying session alive
|
||||||
|
// even if a future CPU-fallback / EP-rebuild path resets the
|
||||||
|
// member here on another thread.
|
||||||
|
std::shared_ptr<ONNXRtmPose> m_ortEngine;
|
||||||
|
|
||||||
|
bool InitOrtEngine();
|
||||||
|
bool InitOrtEngine(ANSCENTER::EngineType engineType);
|
||||||
|
void WarmUpEngine();
|
||||||
|
|
||||||
|
// Per-bbox pipeline used by RunInference. Pulled out as its own
|
||||||
|
// method so the future bbox-overload (ANSODBase::RunInference with
|
||||||
|
// a vector<cv::Rect>) can route here directly.
|
||||||
|
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
|
||||||
|
const std::string& camera_id);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace ANSCENTER
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,702 @@
|
|||||||
|
#include "ANSRTRTMPOSE.h"
|
||||||
|
#include "Utility.h"
|
||||||
|
#include "ANSLicense.h" // ANS_DBG macro for DebugView
|
||||||
|
#include <opencv2/cudaimgproc.hpp>
|
||||||
|
#include <opencv2/cudawarping.hpp>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cmath>
|
||||||
|
|
||||||
|
namespace ANSCENTER {
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSODBase interface — OptimizeModel
|
||||||
|
// ====================================================================
|
||||||
|
bool ANSRTRTMPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: fp16=%d", fp16 ? 1 : 0);
|
||||||
|
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false;
|
||||||
|
if (!FileExist(_modelFilePath)) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::OptimizeModel",
|
||||||
|
"Raw model file path does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: missing onnx '%s'", _modelFilePath.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
_fp16 = fp16;
|
||||||
|
optimizedModelFolder = GetParentFolder(_modelFilePath);
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
|
||||||
|
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
|
||||||
|
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
|
||||||
|
m_options.maxInputHeight = _modelConfig.maxInputHeight;
|
||||||
|
m_options.minInputHeight = _modelConfig.minInputHeight;
|
||||||
|
m_options.optInputHeight = _modelConfig.optInputHeight;
|
||||||
|
m_options.maxInputWidth = _modelConfig.maxInputWidth;
|
||||||
|
m_options.minInputWidth = _modelConfig.minInputWidth;
|
||||||
|
m_options.optInputWidth = _modelConfig.optInputWidth;
|
||||||
|
m_options.engineFileDir = optimizedModelFolder;
|
||||||
|
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
|
||||||
|
m_trtEngine = std::make_shared<Engine<float>>(m_options);
|
||||||
|
}
|
||||||
|
auto succ = m_trtEngine->buildWithRetry(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE);
|
||||||
|
if (!succ) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::OptimizeModel",
|
||||||
|
"Error: Unable to build TensorRT engine. " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: buildWithRetry FAILED for '%s'", _modelFilePath.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: build OK -> '%s'", optimizedModelFolder.c_str());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::OptimizeModel", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSODBase interface — LoadModel
|
||||||
|
// ====================================================================
|
||||||
|
bool ANSRTRTMPOSE::LoadModel(const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword) {
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ModelLoadingGuard mlg(_modelLoading);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModel: zip='%s'", modelZipFilePath.c_str());
|
||||||
|
try {
|
||||||
|
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
|
||||||
|
if (!result) return false;
|
||||||
|
|
||||||
|
_modelConfig.modelType = ModelType::RTRTMPOSE;
|
||||||
|
// RTMPose has fixed input dims — we still propagate them through ModelConfig
|
||||||
|
// so the rest of the pipeline (caching, logging) can introspect them.
|
||||||
|
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = RTM_INPUT_H;
|
||||||
|
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = RTM_INPUT_W;
|
||||||
|
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
|
||||||
|
_modelConfig.numKPS = 26; // default to RTMPose body 2d-26
|
||||||
|
if (_modelConfig.kpsThreshold <= 0)
|
||||||
|
_modelConfig.kpsThreshold = KPS_THRESHOLD;
|
||||||
|
_fp16 = true;
|
||||||
|
KPS_THRESHOLD = _modelConfig.kpsThreshold;
|
||||||
|
|
||||||
|
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
|
||||||
|
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
|
||||||
|
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
|
||||||
|
m_options.maxInputHeight = _modelConfig.maxInputHeight;
|
||||||
|
m_options.minInputHeight = _modelConfig.minInputHeight;
|
||||||
|
m_options.optInputHeight = _modelConfig.optInputHeight;
|
||||||
|
m_options.maxInputWidth = _modelConfig.maxInputWidth;
|
||||||
|
m_options.minInputWidth = _modelConfig.minInputWidth;
|
||||||
|
m_options.optInputWidth = _modelConfig.optInputWidth;
|
||||||
|
m_options.engineFileDir = _modelFolder;
|
||||||
|
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
|
||||||
|
|
||||||
|
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||||
|
if (FileExist(_modelConfigFile)) {
|
||||||
|
ModelType modelType;
|
||||||
|
std::vector<int> inputShape;
|
||||||
|
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||||
|
if (inputShape.size() == 2) {
|
||||||
|
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||||
|
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||||
|
std::ifstream isValid(_classFilePath);
|
||||||
|
if (!isValid) LoadClassesFromString();
|
||||||
|
else LoadClassesFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this->_loadEngineOnCreation) {
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
m_poolKey = { _modelFilePath,
|
||||||
|
static_cast<int>(m_options.precision), m_options.maxBatchSize };
|
||||||
|
m_trtEngine = EnginePoolManager<float>::instance().acquire(
|
||||||
|
m_poolKey, m_options, _modelFilePath,
|
||||||
|
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
|
||||||
|
m_usingSharedPool = (m_trtEngine != nullptr);
|
||||||
|
}
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::LoadModel",
|
||||||
|
"Error: Unable to load TensorRT engine. " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine acquire FAILED '%s'", _modelFilePath.c_str());
|
||||||
|
_modelLoadValid = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize;
|
||||||
|
m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize;
|
||||||
|
m_trtEngine->warmUp();
|
||||||
|
|
||||||
|
// Pull true input geometry from the engine; RTMPose is fixed-size,
|
||||||
|
// but reading it back protects us against unexpected ONNX exports.
|
||||||
|
const auto& inputDims = m_trtEngine->getInputDims();
|
||||||
|
if (!inputDims.empty()) {
|
||||||
|
if (inputDims[0].d[1] > 0) RTM_INPUT_H = inputDims[0].d[1];
|
||||||
|
if (inputDims[0].d[2] > 0) RTM_INPUT_W = inputDims[0].d[2];
|
||||||
|
}
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine ready %dx%d kps=%d sharedPool=%d",
|
||||||
|
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
|
||||||
|
}
|
||||||
|
_modelLoadValid = true;
|
||||||
|
_isInitialized = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::LoadModel", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModel: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSODBase interface — LoadModelFromFolder
|
||||||
|
// ====================================================================
|
||||||
|
bool ANSRTRTMPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
std::string modelName, std::string className,
|
||||||
|
const std::string& modelFolder, std::string& labelMap) {
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ModelLoadingGuard mlg(_modelLoading);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: folder='%s' model='%s'",
|
||||||
|
modelFolder.c_str(), modelName.c_str());
|
||||||
|
try {
|
||||||
|
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig,
|
||||||
|
modelName, className, modelFolder, labelMap);
|
||||||
|
if (!result) return false;
|
||||||
|
|
||||||
|
_modelConfig = modelConfig;
|
||||||
|
_modelConfig.modelType = ModelType::RTRTMPOSE;
|
||||||
|
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = RTM_INPUT_H;
|
||||||
|
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = RTM_INPUT_W;
|
||||||
|
_modelConfig.precisionType = PrecisionType::FP32;
|
||||||
|
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
|
||||||
|
_modelConfig.numKPS = 26;
|
||||||
|
if (_modelConfig.kpsThreshold <= 0)
|
||||||
|
_modelConfig.kpsThreshold = KPS_THRESHOLD;
|
||||||
|
_fp16 = true;
|
||||||
|
KPS_THRESHOLD = _modelConfig.kpsThreshold;
|
||||||
|
|
||||||
|
std::string _modelName = modelName;
|
||||||
|
if (_modelName.empty()) _modelName = "train_last";
|
||||||
|
std::string modelFullName = _modelName + ".onnx";
|
||||||
|
|
||||||
|
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
|
||||||
|
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
|
||||||
|
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
|
||||||
|
m_options.maxInputHeight = _modelConfig.maxInputHeight;
|
||||||
|
m_options.minInputHeight = _modelConfig.minInputHeight;
|
||||||
|
m_options.optInputHeight = _modelConfig.optInputHeight;
|
||||||
|
m_options.maxInputWidth = _modelConfig.maxInputWidth;
|
||||||
|
m_options.minInputWidth = _modelConfig.minInputWidth;
|
||||||
|
m_options.optInputWidth = _modelConfig.optInputWidth;
|
||||||
|
m_options.engineFileDir = _modelFolder;
|
||||||
|
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
|
||||||
|
|
||||||
|
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
|
||||||
|
if (FileExist(_modelConfigFile)) {
|
||||||
|
ModelType modelType;
|
||||||
|
std::vector<int> inputShape;
|
||||||
|
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||||
|
if (inputShape.size() == 2) {
|
||||||
|
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||||
|
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
_classFilePath = CreateFilePath(_modelFolder, className);
|
||||||
|
std::ifstream isValid(_classFilePath);
|
||||||
|
if (!isValid) LoadClassesFromString();
|
||||||
|
else LoadClassesFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
labelMap.clear();
|
||||||
|
if (!_classes.empty())
|
||||||
|
labelMap = VectorToCommaSeparatedString(_classes);
|
||||||
|
|
||||||
|
if (this->_loadEngineOnCreation) {
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
m_poolKey = { _modelFilePath,
|
||||||
|
static_cast<int>(m_options.precision), m_options.maxBatchSize };
|
||||||
|
m_trtEngine = EnginePoolManager<float>::instance().acquire(
|
||||||
|
m_poolKey, m_options, _modelFilePath,
|
||||||
|
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
|
||||||
|
m_usingSharedPool = (m_trtEngine != nullptr);
|
||||||
|
}
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::LoadModelFromFolder",
|
||||||
|
"Error: Unable to load TensorRT engine. " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine acquire FAILED '%s'",
|
||||||
|
_modelFilePath.c_str());
|
||||||
|
_modelLoadValid = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize;
|
||||||
|
m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize;
|
||||||
|
m_trtEngine->warmUp();
|
||||||
|
|
||||||
|
const auto& inputDims = m_trtEngine->getInputDims();
|
||||||
|
if (!inputDims.empty()) {
|
||||||
|
if (inputDims[0].d[1] > 0) RTM_INPUT_H = inputDims[0].d[1];
|
||||||
|
if (inputDims[0].d[2] > 0) RTM_INPUT_W = inputDims[0].d[2];
|
||||||
|
}
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine ready %dx%d kps=%d sharedPool=%d",
|
||||||
|
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
|
||||||
|
}
|
||||||
|
_modelLoadValid = true;
|
||||||
|
_isInitialized = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSODBase interface — Initialize
|
||||||
|
// ====================================================================
|
||||||
|
bool ANSRTRTMPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword,
|
||||||
|
std::string& labelMap) {
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
ModelLoadingGuard mlg(_modelLoading);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Initialize: zip='%s'", modelZipFilePath.c_str());
|
||||||
|
try {
|
||||||
|
const bool engineAlreadyLoaded =
|
||||||
|
_modelLoadValid && _isInitialized && m_trtEngine != nullptr;
|
||||||
|
_modelLoadValid = false;
|
||||||
|
bool result = ANSODBase::Initialize(licenseKey, modelConfig,
|
||||||
|
modelZipFilePath, modelZipPassword, labelMap);
|
||||||
|
if (!result) return false;
|
||||||
|
|
||||||
|
_modelConfig = modelConfig;
|
||||||
|
_modelConfig.modelType = ModelType::RTRTMPOSE;
|
||||||
|
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = RTM_INPUT_H;
|
||||||
|
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = RTM_INPUT_W;
|
||||||
|
_modelConfig.precisionType = PrecisionType::FP32;
|
||||||
|
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
|
||||||
|
_modelConfig.numKPS = 26;
|
||||||
|
if (_modelConfig.kpsThreshold <= 0)
|
||||||
|
_modelConfig.kpsThreshold = KPS_THRESHOLD;
|
||||||
|
_fp16 = true;
|
||||||
|
KPS_THRESHOLD = _modelConfig.kpsThreshold;
|
||||||
|
|
||||||
|
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
|
||||||
|
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
|
||||||
|
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
|
||||||
|
m_options.maxInputHeight = _modelConfig.maxInputHeight;
|
||||||
|
m_options.minInputHeight = _modelConfig.minInputHeight;
|
||||||
|
m_options.optInputHeight = _modelConfig.optInputHeight;
|
||||||
|
m_options.maxInputWidth = _modelConfig.maxInputWidth;
|
||||||
|
m_options.minInputWidth = _modelConfig.minInputWidth;
|
||||||
|
m_options.optInputWidth = _modelConfig.optInputWidth;
|
||||||
|
m_options.engineFileDir = _modelFolder;
|
||||||
|
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
|
||||||
|
|
||||||
|
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||||
|
if (FileExist(_modelConfigFile)) {
|
||||||
|
ModelType modelType;
|
||||||
|
std::vector<int> inputShape;
|
||||||
|
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||||
|
if (inputShape.size() == 2) {
|
||||||
|
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||||
|
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||||
|
std::ifstream isValid(_classFilePath);
|
||||||
|
if (!isValid) LoadClassesFromString();
|
||||||
|
else LoadClassesFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
labelMap.clear();
|
||||||
|
if (!_classes.empty())
|
||||||
|
labelMap = VectorToCommaSeparatedString(_classes);
|
||||||
|
|
||||||
|
if (this->_loadEngineOnCreation && !engineAlreadyLoaded) {
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
m_poolKey = { _modelFilePath,
|
||||||
|
static_cast<int>(m_options.precision), m_options.maxBatchSize };
|
||||||
|
m_trtEngine = EnginePoolManager<float>::instance().acquire(
|
||||||
|
m_poolKey, m_options, _modelFilePath,
|
||||||
|
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
|
||||||
|
m_usingSharedPool = (m_trtEngine != nullptr);
|
||||||
|
}
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::Initialize",
|
||||||
|
"Error: Unable to load TensorRT engine. " + _modelFilePath, __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Initialize: engine acquire FAILED '%s'", _modelFilePath.c_str());
|
||||||
|
_modelLoadValid = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize;
|
||||||
|
m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize;
|
||||||
|
m_trtEngine->warmUp();
|
||||||
|
|
||||||
|
const auto& inputDims = m_trtEngine->getInputDims();
|
||||||
|
if (!inputDims.empty()) {
|
||||||
|
if (inputDims[0].d[1] > 0) RTM_INPUT_H = inputDims[0].d[1];
|
||||||
|
if (inputDims[0].d[2] > 0) RTM_INPUT_W = inputDims[0].d[2];
|
||||||
|
}
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Initialize: engine ready %dx%d kps=%d sharedPool=%d",
|
||||||
|
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
|
||||||
|
}
|
||||||
|
_modelLoadValid = true;
|
||||||
|
_isInitialized = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::Initialize", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Initialize: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// RunInference / Destroy / Destructor
|
||||||
|
// ====================================================================
|
||||||
|
std::vector<Object> ANSRTRTMPOSE::RunInference(const cv::Mat& inputImgBGR) {
|
||||||
|
return RunInference(inputImgBGR, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<Object> ANSRTRTMPOSE::RunInference(const cv::Mat& inputImgBGR,
|
||||||
|
const std::string& camera_id) {
|
||||||
|
if (!PreInferenceCheck("ANSRTRTMPOSE::RunInference")) return {};
|
||||||
|
if (inputImgBGR.empty()) {
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "RunInference: empty input cam='%s'", camera_id.c_str());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
// Top-down pose estimator: with no external bbox, treat the whole
|
||||||
|
// image as a single person crop. Higher-level orchestration that
|
||||||
|
// wants per-person pose should call ANSODBase::RunInference(input,
|
||||||
|
// bboxes, cam) instead — that overload crops each bbox and routes
|
||||||
|
// back through us per crop.
|
||||||
|
const cv::Rect wholeImage(0, 0, inputImgBGR.cols, inputImgBGR.rows);
|
||||||
|
return InferenceOnBox(inputImgBGR, wholeImage, camera_id);
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::RunInference", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "RunInference: EXCEPTION '%s'", e.what());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ANSRTRTMPOSE::~ANSRTRTMPOSE() {
|
||||||
|
try { Destroy(); }
|
||||||
|
catch (std::exception& e) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::~ANSRTRTMPOSE()", e.what(), __FILE__, __LINE__);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ANSRTRTMPOSE::Destroy() {
|
||||||
|
try {
|
||||||
|
if (m_usingSharedPool) {
|
||||||
|
EnginePoolManager<float>::instance().release(m_poolKey);
|
||||||
|
m_trtEngine.reset();
|
||||||
|
m_usingSharedPool = false;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
m_trtEngine.reset();
|
||||||
|
}
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Destroy: engine released");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (std::exception& e) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::Destroy()", e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Destroy: EXCEPTION '%s'", e.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// Per-bbox single-shot inference pipeline
|
||||||
|
// ====================================================================
|
||||||
|
std::vector<Object> ANSRTRTMPOSE::InferenceOnBox(const cv::Mat& input,
|
||||||
|
const cv::Rect& box,
|
||||||
|
const std::string& camera_id) {
|
||||||
|
if (!m_trtEngine) {
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: engine null");
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (box.width <= 0 || box.height <= 0) {
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: invalid box %dx%d", box.width, box.height);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
m_trtEngine->setDeviceContext();
|
||||||
|
|
||||||
|
// Phase 1 — preprocess under lock (affine warp on CPU, upload to GPU)
|
||||||
|
ImageMetadata meta;
|
||||||
|
std::vector<std::vector<cv::cuda::GpuMat>> trtInput;
|
||||||
|
{
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
trtInput = Preprocess(input, box, meta);
|
||||||
|
}
|
||||||
|
if (trtInput.empty() || trtInput[0].empty()) {
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: preprocess failed box=%d,%d %dx%d",
|
||||||
|
box.x, box.y, box.width, box.height);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 2 — inference (mutex released; pool dispatches to idle slot)
|
||||||
|
std::vector<std::vector<std::vector<float>>> featureVectors;
|
||||||
|
if (!m_trtEngine->runInference(trtInput, featureVectors)) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::InferenceOnBox", "runInference failed", __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: runInference FAILED");
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (featureVectors.empty() || featureVectors[0].size() < 2) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::InferenceOnBox",
|
||||||
|
"Expected 2 SimCC outputs (x,y), got " +
|
||||||
|
std::to_string(featureVectors.empty() ? 0 : featureVectors[0].size()),
|
||||||
|
__FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: bad output count outer=%zu inner=%zu",
|
||||||
|
featureVectors.size(),
|
||||||
|
featureVectors.empty() ? size_t{ 0 } : featureVectors[0].size());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 3 — postprocess. Output ordering in featureVectors matches the
|
||||||
|
// engine's output binding order; for RTMPose ONNX exports that's
|
||||||
|
// [simcc_x, simcc_y]. We disambiguate by feature length anyway.
|
||||||
|
const auto& outputDims = m_trtEngine->getOutputDims();
|
||||||
|
if (outputDims.size() < 2 ||
|
||||||
|
outputDims[0].nbDims < 3 || outputDims[1].nbDims < 3) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::InferenceOnBox",
|
||||||
|
"Unexpected output dims (need 2 outputs of rank>=3)", __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: outputDims rank issue count=%zu",
|
||||||
|
outputDims.size());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
const int K0 = outputDims[0].d[1];
|
||||||
|
const int W0 = outputDims[0].d[2];
|
||||||
|
const int K1 = outputDims[1].d[1];
|
||||||
|
const int H1 = outputDims[1].d[2];
|
||||||
|
if (K0 != K1) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::InferenceOnBox",
|
||||||
|
"SimCC keypoint count mismatch x=" + std::to_string(K0) +
|
||||||
|
" y=" + std::to_string(K1), __FILE__, __LINE__);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
// Pick which output is X (longer of the model-input dims) vs Y.
|
||||||
|
// For 192x256 input, ext_w=384 is shorter than ext_h=512, but the order
|
||||||
|
// of the two SimCC outputs in the ONNX is fixed, so we identify by
|
||||||
|
// length proportional to RTM_INPUT_W vs RTM_INPUT_H rather than by
|
||||||
|
// assuming x-first.
|
||||||
|
const std::vector<float>* simcc_x = &featureVectors[0][0];
|
||||||
|
const std::vector<float>* simcc_y = &featureVectors[0][1];
|
||||||
|
int extW = W0;
|
||||||
|
int extH = H1;
|
||||||
|
const float ratioFirst = (RTM_INPUT_W > 0)
|
||||||
|
? static_cast<float>(W0) / static_cast<float>(RTM_INPUT_W) : 0.f;
|
||||||
|
const float ratioSecond = (RTM_INPUT_H > 0)
|
||||||
|
? static_cast<float>(W0) / static_cast<float>(RTM_INPUT_H) : 0.f;
|
||||||
|
// If the first output's last-dim looks like H (proportional to input H,
|
||||||
|
// not W), swap so simcc_x indexes the W-axis.
|
||||||
|
if (std::abs(ratioFirst - 2.f) > std::abs(ratioSecond - 2.f)) {
|
||||||
|
std::swap(simcc_x, simcc_y);
|
||||||
|
extW = H1;
|
||||||
|
extH = W0;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||||
|
Object obj = PostprocessSimCC(*simcc_x, *simcc_y, K0, extW, extH, meta, camera_id);
|
||||||
|
std::vector<Object> out;
|
||||||
|
if (!obj.kps.empty()) out.push_back(std::move(obj));
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: kps=%d cam='%s'",
|
||||||
|
out.empty() ? 0 : K0, camera_id.c_str());
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// Preprocess — affine crop -> upload BGR (Engine handles RGB norm)
|
||||||
|
//
|
||||||
|
// NOTE on color order: Engine<float>::runInference calls blobFromGpuMats
|
||||||
|
// with swapRB=false (see EngineRunInference.inl). So the GpuMat we hand
|
||||||
|
// it is interpreted channel-by-channel as the model input. RTMPose was
|
||||||
|
// trained on RGB, so we cvtColor BGR -> RGB before upload (mirroring
|
||||||
|
// ANSTENSORRTPOSE / ANSRTYOLO).
|
||||||
|
// ====================================================================
|
||||||
|
std::vector<std::vector<cv::cuda::GpuMat>> ANSRTRTMPOSE::Preprocess(
|
||||||
|
const cv::Mat& inputImage, const cv::Rect& box, ImageMetadata& outMeta) {
|
||||||
|
try {
|
||||||
|
if (!_licenseValid) {
|
||||||
|
_logger.LogFatal("ANSRTRTMPOSE::Preprocess", "Invalid license", __FILE__, __LINE__);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
cv::Mat srcImg = inputImage;
|
||||||
|
if (srcImg.channels() == 1) {
|
||||||
|
cv::cvtColor(srcImg, srcImg, cv::COLOR_GRAY2BGR);
|
||||||
|
}
|
||||||
|
outMeta.imgHeight = static_cast<float>(srcImg.rows);
|
||||||
|
outMeta.imgWidth = static_cast<float>(srcImg.cols);
|
||||||
|
outMeta.personBox = box;
|
||||||
|
if (outMeta.imgHeight <= 0 || outMeta.imgWidth <= 0) return {};
|
||||||
|
|
||||||
|
cv::Mat crop;
|
||||||
|
CropImageByDetectBox(srcImg, box, crop, outMeta.affine_rev);
|
||||||
|
if (crop.empty()) {
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Preprocess: warpAffine produced empty crop");
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
cv::Mat rgb;
|
||||||
|
cv::cvtColor(crop, rgb, cv::COLOR_BGR2RGB);
|
||||||
|
|
||||||
|
cv::cuda::Stream stream;
|
||||||
|
cv::cuda::GpuMat gpuRGB;
|
||||||
|
gpuRGB.upload(rgb, stream);
|
||||||
|
stream.waitForCompletion();
|
||||||
|
|
||||||
|
std::vector<cv::cuda::GpuMat> input{ std::move(gpuRGB) };
|
||||||
|
std::vector<std::vector<cv::cuda::GpuMat>> inputs{ std::move(input) };
|
||||||
|
return inputs;
|
||||||
|
}
|
||||||
|
catch (const std::exception& e) {
|
||||||
|
_logger.LogWarn("ANSRTRTMPOSE::Preprocess",
|
||||||
|
std::string("Skipped frame: ") + e.what(), __FILE__, __LINE__);
|
||||||
|
ANS_DBG("ANSRTRTMPOSE", "Preprocess: EXCEPTION '%s'", e.what());
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// PostprocessSimCC
|
||||||
|
//
|
||||||
|
// For each keypoint k:
|
||||||
|
// px = argmax(simcc_x[k]) ; sx = simcc_x[k][px]
|
||||||
|
// py = argmax(simcc_y[k]) ; sy = simcc_y[k][py]
|
||||||
|
// score = max(sx, sy)
|
||||||
|
// raw_x = px * 0.5 ; raw_y = py * 0.5
|
||||||
|
// [x, y]^T = affine_rev * [raw_x, raw_y, 1]^T
|
||||||
|
//
|
||||||
|
// This matches RTMPose::Inference in rtmpose_tensorrt.cpp.
|
||||||
|
// ====================================================================
|
||||||
|
Object ANSRTRTMPOSE::PostprocessSimCC(const std::vector<float>& simcc_x,
|
||||||
|
const std::vector<float>& simcc_y,
|
||||||
|
int numKeypoints, int extW, int extH,
|
||||||
|
const ImageMetadata& meta,
|
||||||
|
const std::string& camera_id) {
|
||||||
|
Object obj{};
|
||||||
|
if (numKeypoints <= 0 || extW <= 0 || extH <= 0) return obj;
|
||||||
|
if (static_cast<int>(simcc_x.size()) < numKeypoints * extW ||
|
||||||
|
static_cast<int>(simcc_y.size()) < numKeypoints * extH) {
|
||||||
|
_logger.LogError("ANSRTRTMPOSE::PostprocessSimCC",
|
||||||
|
"SimCC buffer underflow: x=" + std::to_string(simcc_x.size()) +
|
||||||
|
" y=" + std::to_string(simcc_y.size()) +
|
||||||
|
" expected " + std::to_string(numKeypoints * extW) + "/" +
|
||||||
|
std::to_string(numKeypoints * extH), __FILE__, __LINE__);
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
const double m0 = meta.affine_rev.at<double>(0, 0);
|
||||||
|
const double m1 = meta.affine_rev.at<double>(0, 1);
|
||||||
|
const double m2 = meta.affine_rev.at<double>(0, 2);
|
||||||
|
const double m3 = meta.affine_rev.at<double>(1, 0);
|
||||||
|
const double m4 = meta.affine_rev.at<double>(1, 1);
|
||||||
|
const double m5 = meta.affine_rev.at<double>(1, 2);
|
||||||
|
|
||||||
|
std::vector<float> kps;
|
||||||
|
kps.reserve(numKeypoints * 3);
|
||||||
|
std::vector<cv::Point2f> polygon;
|
||||||
|
polygon.reserve(numKeypoints);
|
||||||
|
std::stringstream xss, yss;
|
||||||
|
float meanScore = 0.f;
|
||||||
|
int meanCount = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < numKeypoints; ++i) {
|
||||||
|
const float* xPtr = simcc_x.data() + static_cast<size_t>(i) * extW;
|
||||||
|
const float* yPtr = simcc_y.data() + static_cast<size_t>(i) * extH;
|
||||||
|
const int px = static_cast<int>(std::max_element(xPtr, xPtr + extW) - xPtr);
|
||||||
|
const int py = static_cast<int>(std::max_element(yPtr, yPtr + extH) - yPtr);
|
||||||
|
const float score = std::max(xPtr[px], yPtr[py]);
|
||||||
|
|
||||||
|
// SimCC oversampling factor 2 — same constant as the reference impl.
|
||||||
|
const double raw_x = px * 0.5;
|
||||||
|
const double raw_y = py * 0.5;
|
||||||
|
|
||||||
|
float worldX = static_cast<float>(m0 * raw_x + m1 * raw_y + m2);
|
||||||
|
float worldY = static_cast<float>(m3 * raw_x + m4 * raw_y + m5);
|
||||||
|
|
||||||
|
// Clamp into image bounds so downstream code can index safely.
|
||||||
|
worldX = std::clamp(worldX, 0.f, meta.imgWidth - 1.f);
|
||||||
|
worldY = std::clamp(worldY, 0.f, meta.imgHeight - 1.f);
|
||||||
|
|
||||||
|
kps.push_back(worldX);
|
||||||
|
kps.push_back(worldY);
|
||||||
|
kps.push_back(score);
|
||||||
|
polygon.emplace_back(worldX, worldY);
|
||||||
|
|
||||||
|
xss << worldX;
|
||||||
|
yss << worldY;
|
||||||
|
if (i + 1 < numKeypoints) { xss << ";"; yss << ";"; }
|
||||||
|
|
||||||
|
if (score >= KPS_THRESHOLD) { meanScore += score; ++meanCount; }
|
||||||
|
}
|
||||||
|
|
||||||
|
obj.classId = 0;
|
||||||
|
obj.className = _classes.empty() ? std::string("person") : _classes.front();
|
||||||
|
obj.confidence = (meanCount > 0) ? meanScore / meanCount : 0.f;
|
||||||
|
obj.box = meta.personBox;
|
||||||
|
obj.kps = std::move(kps);
|
||||||
|
obj.polygon = std::move(polygon);
|
||||||
|
obj.cameraId = camera_id;
|
||||||
|
obj.extraInfo = xss.str() + "|" + yss.str();
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// Affine helpers — direct port of utils.h / rtmpose_tensorrt.cpp
|
||||||
|
// ====================================================================
|
||||||
|
cv::Mat ANSRTRTMPOSE::GetAffineTransform(float center_x, float center_y,
|
||||||
|
float scale_w, float scale_h,
|
||||||
|
int output_w, int output_h, bool inverse) {
|
||||||
|
cv::Point2f src1(center_x, center_y);
|
||||||
|
cv::Point2f src2(center_x, center_y - scale_w * 0.5f);
|
||||||
|
cv::Point2f src3(src2.x - (src1.y - src2.y),
|
||||||
|
src2.y + (src1.x - src2.x));
|
||||||
|
|
||||||
|
const float dcx = output_w * 0.5f;
|
||||||
|
const float dcy = output_h * 0.5f;
|
||||||
|
cv::Point2f dst1(dcx, dcy);
|
||||||
|
cv::Point2f dst2(dcx, dcy - output_w * 0.5f);
|
||||||
|
cv::Point2f dst3(dst2.x - (dst1.y - dst2.y),
|
||||||
|
dst2.y + (dst1.x - dst2.x));
|
||||||
|
|
||||||
|
cv::Point2f srcPts[3] = { src1, src2, src3 };
|
||||||
|
cv::Point2f dstPts[3] = { dst1, dst2, dst3 };
|
||||||
|
return inverse ? cv::getAffineTransform(dstPts, srcPts)
|
||||||
|
: cv::getAffineTransform(srcPts, dstPts);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ANSRTRTMPOSE::CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box,
|
||||||
|
cv::Mat& crop, cv::Mat& affine_rev) {
|
||||||
|
// Aspect-fit the model's W:H (e.g. 192:256) over the bbox, then expand
|
||||||
|
// by 1.2x for context — same heuristic as the reference RTMPose impl.
|
||||||
|
float w = static_cast<float>(box.width);
|
||||||
|
float h = static_cast<float>(box.height);
|
||||||
|
const float cx = box.x + w * 0.5f;
|
||||||
|
const float cy = box.y + h * 0.5f;
|
||||||
|
|
||||||
|
const float aspect = static_cast<float>(RTM_INPUT_W) /
|
||||||
|
static_cast<float>(RTM_INPUT_H);
|
||||||
|
if (w > aspect * h) h = w / aspect;
|
||||||
|
else w = h * aspect;
|
||||||
|
const float scale_w = w * 1.2f;
|
||||||
|
const float scale_h = h * 1.2f;
|
||||||
|
|
||||||
|
cv::Mat affine = GetAffineTransform(cx, cy, scale_w, scale_h,
|
||||||
|
RTM_INPUT_W, RTM_INPUT_H, false);
|
||||||
|
affine_rev = GetAffineTransform(cx, cy, scale_w, scale_h,
|
||||||
|
RTM_INPUT_W, RTM_INPUT_H, true);
|
||||||
|
|
||||||
|
cv::warpAffine(image, crop, affine,
|
||||||
|
cv::Size(RTM_INPUT_W, RTM_INPUT_H), cv::INTER_LINEAR);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace ANSCENTER
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
#ifndef ANSRTRTMPOSE_H
|
||||||
|
#define ANSRTRTMPOSE_H
|
||||||
|
#pragma once
|
||||||
|
#include "ANSEngineCommon.h"
|
||||||
|
#include "engine.h"
|
||||||
|
#include "ANSGpuFrameRegistry.h"
|
||||||
|
#include "engine/EnginePoolManager.h"
|
||||||
|
|
||||||
|
namespace ANSCENTER {
|
||||||
|
|
||||||
|
// ====================================================================
|
||||||
|
// ANSRTRTMPOSE — TensorRT-based RTMPose (SimCC) pose estimator
|
||||||
|
//
|
||||||
|
// Ported from C:\Projects\AIVISION\RTM_POSE\TensorRT\rtmpose_tensorrt.cpp
|
||||||
|
// but uses Engine<float> from TensorRTAPI (shared pool / pinned upload /
|
||||||
|
// multi-GPU dispatch / engine cache) instead of raw nvinfer1.
|
||||||
|
//
|
||||||
|
// RTMPose is a top-down single-person pose estimator:
|
||||||
|
// Input : person crop, 192x256 RGB, ImageNet-normalized
|
||||||
|
// Output : two SimCC heatmaps
|
||||||
|
// simcc_x [1, K, ext_w]
|
||||||
|
// simcc_y [1, K, ext_h]
|
||||||
|
// K = number of keypoints (e.g. 17 / 26 / 133)
|
||||||
|
// ext_w / ext_h = SimCC oversampled dims (typ. 2 * input)
|
||||||
|
//
|
||||||
|
// RunInference(image) treats the whole input as one person crop. To run
|
||||||
|
// on multiple people, call ANSODBase::RunInference(image, bboxes, cam) —
|
||||||
|
// the base class crops each bbox and dispatches to RunInference per crop.
|
||||||
|
// ====================================================================
|
||||||
|
class ANSENGINE_API ANSRTRTMPOSE : public ANSODBase {
|
||||||
|
public:
|
||||||
|
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword,
|
||||||
|
std::string& labelMap) override;
|
||||||
|
|
||||||
|
bool LoadModel(const std::string& modelZipFilePath,
|
||||||
|
const std::string& modelZipPassword) override;
|
||||||
|
|
||||||
|
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
|
||||||
|
std::string modelName, std::string className,
|
||||||
|
const std::string& modelFolder,
|
||||||
|
std::string& labelMap) override;
|
||||||
|
|
||||||
|
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override;
|
||||||
|
|
||||||
|
std::vector<Object> RunInference(const cv::Mat& input);
|
||||||
|
std::vector<Object> RunInference(const cv::Mat& input,
|
||||||
|
const std::string& camera_id);
|
||||||
|
|
||||||
|
bool Destroy();
|
||||||
|
~ANSRTRTMPOSE();
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::string _modelFilePath;
|
||||||
|
bool _modelLoadValid{ false };
|
||||||
|
bool _fp16{ false };
|
||||||
|
int m_maxSlotsPerGpu{ 1 };
|
||||||
|
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
|
||||||
|
|
||||||
|
// Per-call image geometry — captured before inference, consumed at postprocess.
|
||||||
|
// affine_rev maps SimCC output coords (in 192x256 model space) back to
|
||||||
|
// the original image. Carrying this per-call keeps the class re-entrant.
|
||||||
|
struct ImageMetadata {
|
||||||
|
cv::Mat affine_rev;
|
||||||
|
float imgWidth = 0.f;
|
||||||
|
float imgHeight = 0.f;
|
||||||
|
cv::Rect personBox; // bbox the affine was built from
|
||||||
|
};
|
||||||
|
|
||||||
|
// Single-bbox pipeline: affine-crop -> upload -> Engine.runInference -> SimCC decode
|
||||||
|
std::vector<Object> InferenceOnBox(const cv::Mat& input,
|
||||||
|
const cv::Rect& box,
|
||||||
|
const std::string& camera_id);
|
||||||
|
|
||||||
|
// Build the 192x256 RGB crop around `box` and upload to GPU. outMeta
|
||||||
|
// captures the inverse-affine and original image size for postprocess.
|
||||||
|
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage,
|
||||||
|
const cv::Rect& box,
|
||||||
|
ImageMetadata& outMeta);
|
||||||
|
|
||||||
|
// Decode SimCC outputs into one Object with kps in original-image coords.
|
||||||
|
Object PostprocessSimCC(const std::vector<float>& simcc_x,
|
||||||
|
const std::vector<float>& simcc_y,
|
||||||
|
int numKeypoints, int extW, int extH,
|
||||||
|
const ImageMetadata& meta,
|
||||||
|
const std::string& camera_id);
|
||||||
|
|
||||||
|
// Affine helpers — direct port of utils.h::GetAffineTransform and
|
||||||
|
// rtmpose_tensorrt.cpp::CropImageByDetectBox.
|
||||||
|
static cv::Mat GetAffineTransform(float center_x, float center_y,
|
||||||
|
float scale_w, float scale_h,
|
||||||
|
int output_w, int output_h,
|
||||||
|
bool inverse = false);
|
||||||
|
void CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box,
|
||||||
|
cv::Mat& crop, cv::Mat& affine_rev);
|
||||||
|
|
||||||
|
// ── TensorRT engine (shared pool, same pattern as ANSRTYOLO) ─────
|
||||||
|
std::shared_ptr<Engine<float>> m_trtEngine = nullptr;
|
||||||
|
EnginePoolManager<float>::PoolKey m_poolKey;
|
||||||
|
bool m_usingSharedPool = false;
|
||||||
|
|
||||||
|
ANSCENTER::Options m_options;
|
||||||
|
|
||||||
|
// Fixed RTMPose model geometry (overridden from the engine's input dims
|
||||||
|
// at LoadModel time once the network is built).
|
||||||
|
int RTM_INPUT_W{ 192 };
|
||||||
|
int RTM_INPUT_H{ 256 };
|
||||||
|
|
||||||
|
// ImageNet normalization, expressed for Engine<T>::blobFromGpuMats:
|
||||||
|
// blob = (pixel/255 - subVals) / divVals
|
||||||
|
// subVals = mean / 255 ; divVals = std / 255
|
||||||
|
const std::array<float, 3> SUB_VALS{ 0.485f, 0.456f, 0.406f };
|
||||||
|
const std::array<float, 3> DIV_VALS{ 0.229f, 0.224f, 0.225f };
|
||||||
|
const bool NORMALIZE = true;
|
||||||
|
|
||||||
|
// Keypoint confidence floor for output. Mirrors KPS_THRESHOLD from
|
||||||
|
// ANSTENSORRTPOSE — keypoints below threshold are still emitted (the
|
||||||
|
// SimCC max is taken regardless), but caller can read kps[2] (score)
|
||||||
|
// to decide whether to draw them.
|
||||||
|
float KPS_THRESHOLD{ 0.3f };
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace ANSCENTER
|
||||||
|
#endif
|
||||||
@@ -18,6 +18,7 @@ set(ANSOD_SOURCES
|
|||||||
ANSONNXCL.cpp
|
ANSONNXCL.cpp
|
||||||
ANSONNXOBB.cpp
|
ANSONNXOBB.cpp
|
||||||
ANSONNXPOSE.cpp
|
ANSONNXPOSE.cpp
|
||||||
|
ANSONNXRTMPOSE.cpp
|
||||||
ANSONNXSAM3.cpp
|
ANSONNXSAM3.cpp
|
||||||
ANSONNXSEG.cpp
|
ANSONNXSEG.cpp
|
||||||
ANSONNXYOLO.cpp
|
ANSONNXYOLO.cpp
|
||||||
@@ -27,6 +28,7 @@ set(ANSOD_SOURCES
|
|||||||
ANSOVSEG.cpp
|
ANSOVSEG.cpp
|
||||||
ANSPOSE.cpp
|
ANSPOSE.cpp
|
||||||
ANSRTYOLO.cpp
|
ANSRTYOLO.cpp
|
||||||
|
ANSRTRTMPOSE.cpp
|
||||||
ANSSAM.cpp
|
ANSSAM.cpp
|
||||||
ANSSAM3.cpp
|
ANSSAM3.cpp
|
||||||
ANSTENSORRTCL.cpp
|
ANSTENSORRTCL.cpp
|
||||||
|
|||||||
@@ -52,6 +52,8 @@ extern "C" ANSODENGINE_API GpuFrameData** ANSODEngine_GetTlsGpuFrame() {
|
|||||||
#include "ANSONNXSAM3.h"
|
#include "ANSONNXSAM3.h"
|
||||||
#include "ANSRTYOLO.h"
|
#include "ANSRTYOLO.h"
|
||||||
#include "ANSONNXYOLO.h"
|
#include "ANSONNXYOLO.h"
|
||||||
|
#include "ANSRTRTMPOSE.h"
|
||||||
|
#include "ANSONNXRTMPOSE.h"
|
||||||
#include <pipelines/metadata.h>
|
#include <pipelines/metadata.h>
|
||||||
#include <models/input_data.h>
|
#include <models/input_data.h>
|
||||||
#include "utils/visualizer.hpp"
|
#include "utils/visualizer.hpp"
|
||||||
@@ -445,6 +447,11 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase**
|
|||||||
case 22: // RTPOSE
|
case 22: // RTPOSE
|
||||||
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
||||||
break;
|
break;
|
||||||
|
// ── RTMPose (top-down SimCC pose) ────────────────────────────────────
|
||||||
|
case 32: // ONNXRTMPOSE
|
||||||
|
case 33: // RTRTMPOSE
|
||||||
|
modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */;
|
||||||
|
break;
|
||||||
// ── Segmentation ─────────────────────────────────────────────────────
|
// ── Segmentation ─────────────────────────────────────────────────────
|
||||||
case 23: // ONNXSEG
|
case 23: // ONNXSEG
|
||||||
case 24: // RTSEG
|
case 24: // RTSEG
|
||||||
@@ -699,6 +706,20 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase**
|
|||||||
modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO;
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case 32: //ONNXRTMPOSE
|
||||||
|
(*Handle) = new ANSCENTER::ANSONNXRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
break;
|
||||||
|
case 33: //RTRTMPOSE
|
||||||
|
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
||||||
|
(*Handle) = new ANSCENTER::ANSRTRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
(*Handle) = new ANSCENTER::ANSONNXRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
}
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
(*Handle) = new ANSCENTER::ANSFD();
|
(*Handle) = new ANSCENTER::ANSFD();
|
||||||
modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT;
|
modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT;
|
||||||
@@ -803,6 +824,11 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl
|
|||||||
case 22: // RTPOSE
|
case 22: // RTPOSE
|
||||||
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
||||||
break;
|
break;
|
||||||
|
// ── RTMPose (top-down SimCC pose) ────────────────────────────────────
|
||||||
|
case 32: // ONNXRTMPOSE
|
||||||
|
case 33: // RTRTMPOSE
|
||||||
|
modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */;
|
||||||
|
break;
|
||||||
// ── Segmentation ─────────────────────────────────────────────────────
|
// ── Segmentation ─────────────────────────────────────────────────────
|
||||||
case 23: // ONNXSEG
|
case 23: // ONNXSEG
|
||||||
case 24: // RTSEG
|
case 24: // RTSEG
|
||||||
@@ -1066,6 +1092,20 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl
|
|||||||
modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO;
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case 32: //ONNXRTMPOSE
|
||||||
|
(*Handle) = new ANSCENTER::ANSONNXRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
break;
|
||||||
|
case 33: //RTRTMPOSE
|
||||||
|
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
||||||
|
(*Handle) = new ANSCENTER::ANSRTRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
(*Handle) = new ANSCENTER::ANSONNXRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
}
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
(*Handle) = new ANSCENTER::ANSFD();
|
(*Handle) = new ANSCENTER::ANSFD();
|
||||||
modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT;
|
modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT;
|
||||||
@@ -1207,6 +1247,11 @@ extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase**
|
|||||||
case 22: // RTPOSE
|
case 22: // RTPOSE
|
||||||
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
||||||
break;
|
break;
|
||||||
|
// ── RTMPose (top-down SimCC pose) ────────────────────────────────
|
||||||
|
case 32: // ONNXRTMPOSE
|
||||||
|
case 33: // RTRTMPOSE
|
||||||
|
modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */;
|
||||||
|
break;
|
||||||
// ── Segmentation ─────────────────────────────────────────────────
|
// ── Segmentation ─────────────────────────────────────────────────
|
||||||
case 23: // ONNXSEG
|
case 23: // ONNXSEG
|
||||||
case 24: // RTSEG
|
case 24: // RTSEG
|
||||||
@@ -1459,6 +1504,20 @@ extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase**
|
|||||||
modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO;
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case 32: //ONNXRTMPOSE
|
||||||
|
(*Handle) = new ANSCENTER::ANSONNXRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
break;
|
||||||
|
case 33: //RTRTMPOSE
|
||||||
|
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
||||||
|
(*Handle) = new ANSCENTER::ANSRTRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
(*Handle) = new ANSCENTER::ANSONNXRTMPOSE();
|
||||||
|
modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
}
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
(*Handle) = new ANSCENTER::ANSFD();
|
(*Handle) = new ANSCENTER::ANSFD();
|
||||||
modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT;
|
modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT;
|
||||||
@@ -1530,6 +1589,11 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode
|
|||||||
case 22: // RTPOSE
|
case 22: // RTPOSE
|
||||||
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */;
|
||||||
break;
|
break;
|
||||||
|
// ── RTMPose (top-down SimCC pose) ────────────────────────────────
|
||||||
|
case 32: // ONNXRTMPOSE
|
||||||
|
case 33: // RTRTMPOSE
|
||||||
|
modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */;
|
||||||
|
break;
|
||||||
// ── Segmentation ─────────────────────────────────────────────────
|
// ── Segmentation ─────────────────────────────────────────────────
|
||||||
case 23: // ONNXSEG
|
case 23: // ONNXSEG
|
||||||
case 24: // RTSEG
|
case 24: // RTSEG
|
||||||
@@ -1545,7 +1609,7 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (modelType == 31) // If modelType is RTYOLO (31), handle separately.
|
if (modelType == 31) // If modelType is RTYOLO (31), handle separately.
|
||||||
{
|
{
|
||||||
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
||||||
@@ -1596,6 +1660,16 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case 32: Handle = std::make_unique<ANSCENTER::ANSONNXRTMPOSE>(); modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; break;
|
||||||
|
case 33: {
|
||||||
|
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
||||||
|
Handle = std::make_unique<ANSCENTER::ANSRTRTMPOSE>(); modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Handle = std::make_unique<ANSCENTER::ANSONNXRTMPOSE>(); modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default: {
|
default: {
|
||||||
if (modelDetectionType == 0) // classification
|
if (modelDetectionType == 0) // classification
|
||||||
|
|||||||
Reference in New Issue
Block a user