From a43a99834a6599e37cd475c6aa9013d68bcc71b0 Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Thu, 14 May 2026 21:16:35 +1000 Subject: [PATCH] Complete RTMPOSE --- modules/ANSODEngine/dllmain.cpp | 82 ++++--- .../ANSODEngine-UnitTest.cpp | 231 +++++++++++++++++- 2 files changed, 268 insertions(+), 45 deletions(-) diff --git a/modules/ANSODEngine/dllmain.cpp b/modules/ANSODEngine/dllmain.cpp index b42ba9d..17de06c 100644 --- a/modules/ANSODEngine/dllmain.cpp +++ b/modules/ANSODEngine/dllmain.cpp @@ -443,15 +443,20 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase** modelType = onNvidia ? 31 /* RTYOLO */ : 30 /* ONNXYOLO */; break; // ── Pose ───────────────────────────────────────────────────────────── - case 21: // ONNXPOSE - case 22: // RTPOSE - modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; - break; + // Intentionally NOT auto-routed. The user picks the backend explicitly: + // 21 → ANSONNXPOSE (ONNX Runtime) + // 22 → ANSTENSORRTPOSE (TensorRT, falls back to ONNX on non-NVIDIA + // via the case-22 body) + // Removing the rewrite makes 21 vs 22 comparisons meaningful — the + // previous behavior silently upgraded 21 → 22 on any NVIDIA box. // ── RTMPose (top-down SimCC pose) ──────────────────────────────────── - case 32: // ONNXRTMPOSE - case 33: // RTRTMPOSE - modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; - break; + // Intentionally NOT auto-routed (unlike 21/22 ONNXPOSE/RTPOSE). + // The user picks the backend explicitly: + // 32 → ANSONNXRTMPOSE (ONNX Runtime) + // 33 → ANSRTRTMPOSE (TensorRT, falls back to ONNX on non-NVIDIA + // via the intra-case check inside case 33) + // This keeps 32 vs 33 comparisons meaningful — without it, picking + // 32 on an NVIDIA box silently gets rewritten to 33 here. // ── Segmentation ───────────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -820,15 +825,20 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl modelType = onNvidia ? 31 /* RTYOLO */ : 30 /* ONNXYOLO */; break; // ── Pose ───────────────────────────────────────────────────────────── - case 21: // ONNXPOSE - case 22: // RTPOSE - modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; - break; + // Intentionally NOT auto-routed. The user picks the backend explicitly: + // 21 → ANSONNXPOSE (ONNX Runtime) + // 22 → ANSTENSORRTPOSE (TensorRT, falls back to ONNX on non-NVIDIA + // via the case-22 body) + // Removing the rewrite makes 21 vs 22 comparisons meaningful — the + // previous behavior silently upgraded 21 → 22 on any NVIDIA box. // ── RTMPose (top-down SimCC pose) ──────────────────────────────────── - case 32: // ONNXRTMPOSE - case 33: // RTRTMPOSE - modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; - break; + // Intentionally NOT auto-routed (unlike 21/22 ONNXPOSE/RTPOSE). + // The user picks the backend explicitly: + // 32 → ANSONNXRTMPOSE (ONNX Runtime) + // 33 → ANSRTRTMPOSE (TensorRT, falls back to ONNX on non-NVIDIA + // via the intra-case check inside case 33) + // This keeps 32 vs 33 comparisons meaningful — without it, picking + // 32 on an NVIDIA box silently gets rewritten to 33 here. // ── Segmentation ───────────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -1243,15 +1253,19 @@ extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase** modelType = onNvidia ? 31 /* RTYOLO */ : 30 /* ONNXYOLO */; break; // ── Pose ───────────────────────────────────────────────────────── - case 21: // ONNXPOSE - case 22: // RTPOSE - modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; - break; + // Intentionally NOT auto-routed. The user picks the backend explicitly: + // 21 → ANSONNXPOSE (ONNX Runtime) + // 22 → ANSTENSORRTPOSE (TensorRT, falls back to ONNX on + // non-NVIDIA via the case-22 body) + // Removing the rewrite makes 21 vs 22 comparisons meaningful. // ── RTMPose (top-down SimCC pose) ──────────────────────────────── - case 32: // ONNXRTMPOSE - case 33: // RTRTMPOSE - modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; - break; + // Intentionally NOT auto-routed (unlike 21/22 ONNXPOSE/RTPOSE). + // The user picks the backend explicitly: + // 32 → ANSONNXRTMPOSE (ONNX Runtime) + // 33 → ANSRTRTMPOSE (TensorRT, falls back to ONNX on + // non-NVIDIA via the case-33 body) + // This keeps 32 vs 33 comparisons meaningful — without it, + // picking 32 on an NVIDIA box silently gets rewritten to 33. // ── Segmentation ───────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -1585,15 +1599,19 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode modelType = onNvidia ? 31 /* RTYOLO */ : 30 /* ONNXYOLO */; break; // ── Pose ───────────────────────────────────────────────────────── - case 21: // ONNXPOSE - case 22: // RTPOSE - modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; - break; + // Intentionally NOT auto-routed. The user picks the backend explicitly: + // 21 → ANSONNXPOSE (ONNX Runtime) + // 22 → ANSTENSORRTPOSE (TensorRT, falls back to ONNX on + // non-NVIDIA via the case-22 body) + // Removing the rewrite makes 21 vs 22 comparisons meaningful. // ── RTMPose (top-down SimCC pose) ──────────────────────────────── - case 32: // ONNXRTMPOSE - case 33: // RTRTMPOSE - modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; - break; + // Intentionally NOT auto-routed (unlike 21/22 ONNXPOSE/RTPOSE). + // The user picks the backend explicitly: + // 32 → ANSONNXRTMPOSE (ONNX Runtime) + // 33 → ANSRTRTMPOSE (TensorRT, falls back to ONNX on + // non-NVIDIA via the case-33 body) + // This keeps 32 vs 33 comparisons meaningful — without it, + // picking 32 on an NVIDIA box silently gets rewritten to 33. // ── Segmentation ───────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG diff --git a/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp b/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp index eae0eab..ce79d7c 100644 --- a/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp +++ b/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp @@ -947,8 +947,9 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in } unsigned int bufferLength = 0; - //cv::Mat croppedFrame = frame(cv::Rect(0, 0, 1920, 1080)); - auto start = std::chrono::system_clock::now(); + + // ── Phase 1: JPEG encode ─────────────────────────────────────── + auto t_enc_start = std::chrono::high_resolution_clock::now(); std::vector imageData; bool success = cv::imencode(".jpg", frame, imageData); if (!success) { @@ -956,19 +957,27 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in break; } std::string jpegImage(imageData.begin(), imageData.end()); - /* auto end1 = std::chrono::system_clock::now(); - auto elapsed1 = std::chrono::duration_cast(end1 - start1); - printf("Jpeg conversion Time = %lld ms\n", static_cast(elapsed1.count()));*/ + auto t_enc_end = std::chrono::high_resolution_clock::now(); int height = frame.rows; int width = frame.cols; - //auto start = std::chrono::system_clock::now(); - /* measured work */ - std::string detectionResult = RunInferenceFromJpegString(&infHandle, jpegImage.c_str(), imageData.size(), "ID");// , 100, 100, 0.2); - auto end = std::chrono::system_clock::now(); - auto elapsed = std::chrono::duration_cast(end - start); + + // ── Phase 2: inference only (JPEG decode + detector + pose + + // JSON serialization happen inside) ─ + auto t_inf_start = std::chrono::high_resolution_clock::now(); + std::string detectionResult = RunInferenceFromJpegString(&infHandle, jpegImage.c_str(), imageData.size(), "ID"); + auto t_inf_end = std::chrono::high_resolution_clock::now(); + + auto ms_d = [](auto a, auto b) { + return std::chrono::duration(b - a).count(); + }; + const double encMs = ms_d(t_enc_start, t_enc_end); + const double infMs = ms_d(t_inf_start, t_inf_end); + const double totalMs = encMs + infMs; + frameNum++; - printf("Inference Time = %lld ms\n", static_cast(elapsed.count())); + printf("Frame %d: JPEG=%6.2f ms | Inference=%6.2f ms | Total=%6.2f ms\n", + frameNum, encMs, infMs, totalMs); if (!detectionResult.empty()) { // ── COCO 17-keypoint skeleton (Ultralytics YOLO Pose format) ── @@ -1996,10 +2005,206 @@ int FallDetectorTest() { return 0; } +// ============================================================================ +// VideoDetectorEngine_CPP — JPEG-bypass variant of VideoDetectorEngine. +// +// Differences vs. the JPEG version (only 3 actual lines change): +// • No cv::imencode / no JPEG round-trip +// • Calls RunInference_CPP (cv::Mat** in, vector& out) instead of +// RunInferenceFromJpegString +// • No JSON parsing — iterate vector directly +// +// Everything else (skeleton tables, drawing, auto-detect format) is identical. +// Trimmed down to focus on the data path; tracker is left enabled like the +// original. Call RTMPOSETest_CPP() from main() to compare timings. +// ============================================================================ +int VideoDetectorEngine_CPP(std::string modelFilePath, std::string videoFilePath, + int engineType = 32, int modelDetectionType = 1) { + std::vector classes; + ANSCENTER::ANSODBase* infHandle = nullptr; + std::string labelMap; + std::string licenseKey = ""; + std::string modelZipFilePassword = ""; + float detectionScoreThreshold = 0.3f; + float modelConfThreshold = 0.48f; + float modelNMSThreshold = 0.5f; + int modelType = engineType; + int detectorType = modelDetectionType; + + // ── Optimize + create handle (identical to the JPEG variant) ──────────── + std::cout << "Optimizing model, please wait...." << std::endl; + std::string optmizedModelFolder; + OptimizeModelStr(modelFilePath.c_str(), modelZipFilePassword.c_str(), + modelType, modelDetectionType, 1, optmizedModelFolder); + std::cout << "Optimized model folder: " << optmizedModelFolder << std::endl; + + labelMap = CreateANSODHandle(&infHandle, licenseKey.c_str(), modelFilePath.c_str(), + modelZipFilePassword.c_str(), + detectionScoreThreshold, modelConfThreshold, modelNMSThreshold, + 1, modelType, detectorType, 1); + { + std::stringstream ss(labelMap); + std::string token; + while (std::getline(ss, token, ',')) { + if (!token.empty()) classes.push_back(token); + } + } + std::cout << "classes: " << classes.size() << std::endl; + + SetTracker(&infHandle, 0, 1); // 0 = BYTETRACK + + // ── Skeleton tables (lifted verbatim from VideoDetectorEngine) ────────── + const std::vector> COCO17_SKELETON = { + {0,1},{0,2},{1,3},{2,4},{3,5},{4,6}, + {5,7},{7,9},{6,8},{8,10}, + {5,6},{5,11},{6,12},{11,12}, + {11,13},{13,15},{12,14},{14,16} + }; + const std::vector COCO17_COLORS = { + {255,0,0},{255,0,0},{255,0,0},{255,0,0}, + {0,255,255},{0,255,255}, + {0,255,0},{0,255,0},{0,165,255},{0,165,255}, + {255,255,0},{255,0,255},{255,0,255},{255,255,0}, + {0,255,0},{0,255,0},{0,165,255},{0,165,255} + }; + const std::vector> HALPE26_SKELETON = { + {0,1},{0,2},{1,3},{2,4}, + {17,18},{5,18},{6,18}, + {18,19},{19,11},{19,12}, + {5,7},{7,9},{6,8},{8,10}, + {11,13},{13,15},{12,14},{14,16}, + {15,20},{15,22},{15,24},{16,21},{16,23},{16,25} + }; + const std::vector HALPE26_COLORS = { + {255,0,0},{255,0,0},{255,0,0},{255,0,0}, + {0,255,255},{0,255,255},{0,255,255}, + {255,255,0},{255,255,0},{255,255,0}, + {0,255,0},{0,255,0},{0,165,255},{0,165,255}, + {0,255,0},{0,255,0},{0,165,255},{0,165,255}, + {200,100,255},{200,100,255},{200,100,255}, + {255,100,100},{255,100,100},{255,100,100} + }; + const std::vector> OPENPOSE18_SKELETON = { + {1,2},{2,3},{3,4},{1,5},{5,6},{6,7}, + {1,8},{8,9},{9,10},{1,11},{11,12},{12,13}, + {1,0},{0,14},{14,16},{0,15},{15,17} + }; + const std::vector OPENPOSE18_COLORS = { + {0,165,255},{0,165,255},{0,165,255},{0,255,0},{0,255,0},{0,255,0}, + {0,165,255},{0,165,255},{0,165,255},{0,255,0},{0,255,0},{0,255,0}, + {255,255,0},{255,0,0},{255,0,0},{255,0,0},{255,0,0} + }; + + cv::VideoCapture capture(videoFilePath); + if (!capture.isOpened()) { + std::cout << "Could not read video file: " << videoFilePath << std::endl; + ReleaseANSODHandle(&infHandle); + return -1; + } + + int frameNum = 0; + while (true) { + cv::Mat frame; + if (!capture.read(frame)) { + capture.set(cv::CAP_PROP_POS_FRAMES, 0); + frameNum = 0; + continue; + } + frameNum++; + + // ── THE KEY CHANGE — direct cv::Mat path, no JPEG, no JSON ────────── + // Compare the JPEG-version (~5–10 ms encode + ~5 ms decode + JSON + // serialize/parse round-trip) against this single in-memory call. + std::vector detectionObjects; + cv::Mat* framePtr = &frame; + + auto t0 = std::chrono::high_resolution_clock::now(); + RunInference_CPP(&infHandle, &framePtr, "ID", detectionObjects); + auto t1 = std::chrono::high_resolution_clock::now(); + const double infMs = std::chrono::duration(t1 - t0).count(); + // ──────────────────────────────────────────────────────────────────── + + printf("Frame %d: Inference=%6.2f ms | %zu detections (cv::Mat direct)\n", + frameNum, infMs, detectionObjects.size()); + + // ── Render each Object (no JSON parse — fields are already typed) ─── + for (const auto& obj : detectionObjects) { + const cv::Rect bbox = obj.box; + cv::rectangle(frame, bbox, cv::Scalar(0, 255, 0), 2); + + const std::string label = cv::format("%s:%d:%.2f:%d", + obj.className.c_str(), obj.classId, obj.confidence, obj.trackId); + cv::putText(frame, label, cv::Point(bbox.x, std::max(0, bbox.y - 5)), + cv::FONT_HERSHEY_SIMPLEX, 0.6, cv::Scalar(0, 0, 255), 1, cv::LINE_AA); + + // Auto-detect kps layout from buffer length (same logic as the + // JPEG variant — see VideoDetectorEngine for the rationale). + const auto& kps = obj.kps; + if (kps.empty()) continue; + const size_t n = kps.size(); + + const std::vector>* skeleton = nullptr; + const std::vector* colors = nullptr; + int stride = 3, numKp = 0; + bool hasScore = true; + if (n == 51) { stride=3; numKp=17; skeleton=&COCO17_SKELETON; colors=&COCO17_COLORS; } + else if (n == 78) { stride=3; numKp=26; skeleton=&HALPE26_SKELETON; colors=&HALPE26_COLORS; } + else if (n == 36) { stride=2; numKp=18; skeleton=&OPENPOSE18_SKELETON; colors=&OPENPOSE18_COLORS; + hasScore=false; } + else if (n % 3 == 0) { stride=3; numKp = static_cast(n)/3; } + else if (n % 2 == 0) { stride=2; numKp = static_cast(n)/2; hasScore=false; } + + std::vector pts(numKp); + std::vector conf(numKp, 1.f); + for (int k = 0; k < numKp; ++k) { + const float kx = kps[k * stride]; + const float ky = kps[k * stride + 1]; + pts[k] = cv::Point2f(kx, ky); + conf[k] = hasScore ? kps[k * stride + 2] + : ((kx > 0.f && ky > 0.f) ? 1.f : 0.f); + } + const float kpThr = 0.3f; + if (skeleton) { + for (size_t s = 0; s < skeleton->size(); ++s) { + int i1 = (*skeleton)[s].first, i2 = (*skeleton)[s].second; + if (i1 < numKp && i2 < numKp && conf[i1] > kpThr && conf[i2] > kpThr) { + cv::Scalar c = (s < colors->size()) ? (*colors)[s] : cv::Scalar(200,200,200); + cv::line(frame, pts[i1], pts[i2], c, 2, cv::LINE_AA); + } + } + } + for (int k = 0; k < numKp; ++k) { + if (conf[k] > kpThr) { + cv::circle(frame, pts[k], 4, cv::Scalar(0, 0, 255), -1, cv::LINE_AA); + cv::circle(frame, pts[k], 4, cv::Scalar(255, 255, 255), 1, cv::LINE_AA); + } + } + } + + cv::putText(frame, cv::format("Frame: %d (cv::Mat path)", frameNum), + cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 0.8, + cv::Scalar(255, 255, 255), 2); + cv::imshow("ANS Pose (cv::Mat direct)", frame); + if (cv::waitKey(1) == 27) break; // ESC to quit + } + + ReleaseANSODHandle(&infHandle); + cv::destroyAllWindows(); + return 0; +} + +int RTMPOSETest_CPP() { + int modelType = 32; // ONNXRTMPOSE (use 33 for RTRTMPOSE) + std::string modelFilePath = "E:\\Programs\\DemoAssets\\ANSAIModels\\PoseEstimationModels\\ANSRTMPOSE.zip"; + std::string videoFile = "E:\\Programs\\DemoAssets\\Videos\\rtmpose.mp4"; + VideoDetectorEngine_CPP(modelFilePath, videoFile, modelType); + return 0; +} + int RTMPOSETest() { - int modelType = 32; // ONNX YOLO (30) RT YOLO (31) + int modelType = 33; // ONNX YOLO (30) RT YOLO (31) std::string onnxclmodelFilePath = "E:\\Programs\\DemoAssets\\ANSAIModels\\PoseEstimationModels\\ANSRTMPOSE.zip"; - std::string videoFile = "E:\\Programs\\DemoAssets\\Videos\\ANSOffice.mp4";//"E:\\Programs\\DemoAssets\\Videos\\rtmpose.mp4";// + std::string videoFile ="E:\\Programs\\DemoAssets\\Videos\\rtmpose.mp4";// "E:\\Programs\\DemoAssets\\Videos\\ANSOffice.mp4";// VideoDetectorEngine(onnxclmodelFilePath, videoFile, modelType);// 22 is RT, 21 is onnx return 0; }