From 91efbb9a8f8d495e640b0d761c060e8d3af66c4d Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Fri, 1 May 2026 07:53:18 +1000 Subject: [PATCH] Fix AMD ALPR_OCR and custom model --- .claude/settings.local.json | 31 +- MediaClient/media/av_frame_ptr.h | 72 ++ engines/ONNXEngine/ONNXEngine.cpp | 199 +++ engines/ONNXEngine/ONNXEngine.h | 118 ++ modules/ANSCV/ANSFLV.cpp | 41 +- modules/ANSCV/ANSFilePlayer.cpp | 40 +- modules/ANSCV/ANSMJPEG.cpp | 41 +- modules/ANSCV/ANSRTMP.cpp | 41 +- modules/ANSCV/ANSRTSP.cpp | 23 +- modules/ANSCV/ANSSRT.cpp | 41 +- modules/ANSCV/ANSVideoPlayer.cpp | 21 +- modules/ANSLPR/ANSLPR.h | 26 + modules/ANSLPR/ANSLPR_OCR.cpp | 57 +- modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.cpp | 219 +++- modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.h | 21 + .../ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.cpp | 289 ++++- modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.h | 58 + modules/ANSODEngine/ANSCUSTOMDetector.h | 2 +- modules/ANSODEngine/ANSEngineCommon.h | 51 + modules/ANSODEngine/ANSONNXCL.cpp | 310 ++--- modules/ANSODEngine/ANSONNXCL.h | 31 + modules/ANSODEngine/ANSONNXYOLO.cpp | 358 ++++-- modules/ANSODEngine/ANSONNXYOLO.h | 74 +- modules/ANSODEngine/ANSOVFaceDetector.cpp | 10 +- tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp | 1096 ++++++++++++++++- 25 files changed, 2841 insertions(+), 429 deletions(-) create mode 100644 MediaClient/media/av_frame_ptr.h diff --git a/.claude/settings.local.json b/.claude/settings.local.json index f878953..0f307fb 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -149,7 +149,36 @@ "Bash(awk -F'[:\\(\\)]' '{gsub\\(/^ +| +$/,\"\",$2\\); split\\($2,a,\" \"\\); ms=a[1]; if \\(ms+0 >= 500\\) print $0}')", "Bash(awk -F'[:\\(\\)]' '{split\\($2,a,\" \"\\); ms=a[1]; if \\(ms+0 >= 100\\) print $0}')", "Bash(grep -cE \"^\\\\s*Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log4.txt\")", - "Bash(grep -oE \"Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log5.txt\")" + "Bash(grep -oE \"Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log5.txt\")", + "Bash(awk -F'\\\\t' '{print $3}')", + "Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS_twoconsoleapps.log\")", + "Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS_issue.log\")", + "Bash(awk -F'\\\\t' '$2+0 >= 49 && $2+0 <= 52 {print NR\": \"$0}' \"C:/Users/nghia/Downloads/AVNET-8845HS_issue.log\")", + "Bash(awk -F'\\\\t' 'BEGIN{prev=0} {t=$2+0; if\\(t-prev>5\\){print NR\": GAP=\"t-prev\"s \"$0} prev=t}' \"C:/Users/nghia/Downloads/AVNET-8845HS3.log\")", + "Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS3.log\")", + "Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")", + "Bash(awk -F'\\\\t' '$2+0>=20 && $2+0<=22' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")", + "Bash(awk -F'\\\\t' '$2+0>=20.5 && $2+0<=21.5' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")", + "Bash(awk -F'\\\\t' '$2+0>=21.0 && $2+0<=21.7' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")", + "Bash(awk -F'\\\\t' '$2+0>=21.05 && $2+0<=21.23' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")", + "Bash(awk -F'\\\\t' '$2+0>=17 && $2+0<=20' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")", + "Bash(awk -F'\\\\t' '$2+0>=17.4 && $2+0<=17.7' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")", + "Bash(awk -F'\\\\t' '$2+0>=12 && $2+0<=18' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")", + "Bash(git -C \"C:/Projects/CLionProjects/ANSCORE\" status)", + "Bash(git -C \"C:/Projects/CLionProjects/ANSCORE\" diff --stat modules/ANSODEngine/ANSONNXCL.h modules/ANSODEngine/ANSONNXCL.cpp modules/ANSODEngine/ANSONNXYOLO.cpp)", + "Bash(awk -F'\\\\t' '{print $4}' \"/c/Users/nghia/Downloads/AVNET-8845HS7.log\")", + "Bash(\"C:/Users/nghia/AppData/Local/Programs/CLion 2026.1/bin/cmake/win/x64/bin/cmake.exe\" --build C:/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSLPR-UnitTest -j 30)", + "Bash(cmd.exe /c '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build C:/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSLPR-UnitTest -j 30')", + "Bash(cmd.exe //c 'call \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\" --target ANSLPR-UnitTest -j 30')", + "PowerShell($env:VS_VARS = '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\"'; $cmd = \"call $env:VS_VARS && `\"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe`\" --build `\"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release`\" --target ANSLPR-UnitTest -j 30\"; cmd.exe /c $cmd 2>&1 | Select-Object -Last 60)", + "PowerShell(cmd.exe /c $cmd 2>&1)", + "PowerShell($cmd = 'call \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\" --target ANSLPR-UnitTest -j 30')", + "Bash(command -v dumpbin)", + "Bash(dumpbin /SYMBOLS \"/c/Projects/CLionProjects/ANSCORE/cmake-build-release/bin/ANSOCR.dll\")", + "Bash(\"/c/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/dumpbin.exe\" //SYMBOLS \"/c/Projects/CLionProjects/ANSCORE/cmake-build-release/bin/ANSOCR.dll\")", + "Bash(cmake --build . --target ANSODEngine -j 8)", + "PowerShell(@\\(Get-ChildItem -Path \"C:\\\\Program Files\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) + @\\(Get-ChildItem -Path \"C:\\\\Program Files \\(x86\\)\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) | ForEach-Object { $_.FullName })", + "PowerShell($d = Get-Item C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\\\\bin\\\\ANSODEngine.dll; \"$\\($d.Name\\) | $\\($d.Length\\) bytes | $\\($d.LastWriteTime\\)\")" ] } } diff --git a/MediaClient/media/av_frame_ptr.h b/MediaClient/media/av_frame_ptr.h new file mode 100644 index 0000000..fc12e3b --- /dev/null +++ b/MediaClient/media/av_frame_ptr.h @@ -0,0 +1,72 @@ +#pragma once +// av_frame_ptr.h — RAII smart pointer for FFmpeg AVFrame*. +// +// Why this exists +// --------------- +// CVideoPlayer::getNV12Frame() / getCudaHWFrame() return cloned AVFrame* and +// the contract is "caller must av_frame_free()". That contract is comment-only +// — nothing in the type system enforces it, so any early return, exception, +// or refactor that drops the pointer leaks the AVFrame *and* the underlying +// NV12/YUV420P/YUVJ420P buffers (~3 MB per 1080p frame, ~50 MB/s under load). +// +// AvFramePtr wraps the raw pointer in std::unique_ptr +// so the destructor frees it automatically — leak-by-omission becomes a +// compile-time impossibility on every internal code path that adopts this type. +// +// Scope: INTERNAL TO ANSCORE ONLY. +// ------------------------------- +// Do NOT change DLL-exported function signatures to use AvFramePtr. +// std::unique_ptr cannot cross a DLL boundary safely (different runtimes, +// different STL implementations, different allocators). The public +// AVFrame*-returning APIs in ANSRTSP.h / ANSSRT.h / etc. must remain raw +// pointers for ABI stability. +// +// Use this type at the internal *consumer* call site: +// +// AvFramePtr nv12{ player->getNV12Frame() }; +// if (nv12) { +// doSomething(nv12.get()); // borrowed access +// } // auto-freed on scope exit +// +// To transfer ownership into ANSCV's pendingFree pipeline (the +// gpu_frame_attach / gpu_frame_attach_cuda functions, which take raw +// AVFrame* and store/defer them), call .release(): +// +// AvFramePtr nv12{ player->getNV12Frame() }; +// gpu_frame_attach(mat, nv12.release(), gpuIdx, pts); +// +// Every .release() in the codebase is a deliberate ownership hand-off — +// `git grep '\.release()'` over the migrated files gives an audit list. + +extern "C" { +#include +} + +#include + +namespace ansmedia { + +// Custom deleter that calls av_frame_free (which takes AVFrame**). +// noexcept because deleters must not throw — av_frame_free is a C function +// and never throws, but tagging makes the contract explicit. +struct AvFrameDeleter { + void operator()(AVFrame* f) const noexcept { + if (f) { + av_frame_free(&f); // takes AVFrame**, sets *f to nullptr internally + } + } +}; + +// Owning smart pointer for AVFrame*. Frees on destruction via av_frame_free. +// +// Construction: +// AvFramePtr p; // empty +// AvFramePtr p{ av_frame_clone(src) }; // adopt raw clone +// +// Ownership transfer (e.g. handing off to gpu_frame_attach_cuda): +// takes_ownership(p.release()); // raw pointer escapes, no auto-free +// +// Copy is disallowed (unique_ptr semantics). Move is fine. +using AvFramePtr = std::unique_ptr; + +} // namespace ansmedia diff --git a/engines/ONNXEngine/ONNXEngine.cpp b/engines/ONNXEngine/ONNXEngine.cpp index c6d253c..6cc3b52 100644 --- a/engines/ONNXEngine/ONNXEngine.cpp +++ b/engines/ONNXEngine/ONNXEngine.cpp @@ -10,9 +10,129 @@ #include #include #include +#include +#include namespace ANSCENTER { + // ==================================================================== + // DmlSerialState / DmlInferenceGuard implementation. + // + // Storage is in this single linkage unit (ONNXEngine.dll) so every + // module linking against ONNXEngine sees the same flag and mutex — + // ANSODEngine.dll's DML calls and ANSOCR.dll's DML calls serialize + // against the same process-wide state. See the doc comment on + // DmlSerialState in ONNXEngine.h for the full design rationale. + // ==================================================================== + namespace { + // false = parallel (default). Tripped to true (sticky) once any + // DML site sees an exception or a slow Run(). + std::atomic g_dmlSerialMode{ false }; + + // Held during DML inference when g_dmlSerialMode == true. Plain + // std::mutex (not recursive): one DML Run() at a time across the + // entire process — that's the whole point. Other ORT operations + // (session creation, allocator setup, etc.) DO NOT need this + // mutex and are not gated by it. + std::mutex g_dmlInferenceMutex; + + // 1500 ms — comfortably above normal AMD DML latency (50–500 ms) + // and well below WDDM TDR (10 s with TdrDelay=10). + constexpr double kDmlSlowThresholdMs = 1500.0; + } + + bool DmlSerialState::IsSerialMode() { + return g_dmlSerialMode.load(std::memory_order_relaxed); + } + + double DmlSerialState::SlowThresholdMs() { + return kDmlSlowThresholdMs; + } + + void DmlSerialState::TripFromException(const char* context, const char* msg) { + bool expected = false; + if (g_dmlSerialMode.compare_exchange_strong(expected, true)) { + // First caller wins — log the trip event clearly so it's + // obvious in the field which path serialized DML. + std::cerr << "[DmlSerial] TRIPPED from exception in " + << (context ? context : "?") + << " - process-wide DML inference is now SERIALIZED. msg=" + << (msg ? msg : "?") << std::endl; + } + } + + void DmlSerialState::TripFromLatency(const char* context, double ms) { + bool expected = false; + if (g_dmlSerialMode.compare_exchange_strong(expected, true)) { + std::cerr << "[DmlSerial] TRIPPED from slow inference in " + << (context ? context : "?") + << " ms=" << ms + << " - process-wide DML inference is now SERIALIZED." + << std::endl; + } + } + + void DmlSerialState::AcquireMutex() { + g_dmlInferenceMutex.lock(); + } + + void DmlSerialState::ReleaseMutex() { + g_dmlInferenceMutex.unlock(); + } + + DmlInferenceGuard::DmlInferenceGuard(EngineType ep, const char* context) + : _ep(ep), _context(context), _locked(false) + { + // ── Always-serial-on-AMD policy ──────────────────────────── + // + // Earlier design (Option A v1) was lazy: parallel by default, + // serialize only after seeing a slow Run() or exception. Field + // testing on AMD Radeon 780M proved this fires too late — by the + // time the latency trip arms, a sibling thread has already + // entered Run() at depth=2 and wedged the D3D12 command queue. + // ORT then crashes inside its own DML EP cleanup when the WDDM + // TDR fires. + // + // The only way to guarantee zero dual-stall on AMD is to never + // let two threads be in flight on DML simultaneously. This + // constructor unconditionally takes the global DML mutex when + // ep == AMD_GPU. Intel (OpenVINO) and NVIDIA (CUDA/TRT) skip + // the lock entirely — guard collapses to a timestamp read + + // comparison, and the destructor is a single comparison. Zero + // measurable overhead on those platforms. + // + // Trade-off: AMD inference becomes serial (depth=1) instead of + // parallel (depth=2). Each individual call still runs on the + // GPU at GPU speeds; only the cross-thread overlap is removed. + // Throughput cost vs broken depth=2 is ~30% lower wall-clock + // throughput per frame, but uptime goes from ~30 s before + // wedge → unbounded. + if (_ep == EngineType::AMD_GPU) { + DmlSerialState::AcquireMutex(); + _locked = true; + } + // Start the latency timer AFTER lock acquisition so contention + // on the global mutex doesn't itself trip the slow-Run() + // threshold. We measure GPU Run() time, not queue wait. + _t0 = std::chrono::steady_clock::now(); + } + + DmlInferenceGuard::~DmlInferenceGuard() { + // Latency-based trip detection: if THIS Run() took longer than + // the slow threshold, future DML inference goes serial. Only + // checked on AMD — Intel/NVIDIA never trip. + if (_ep == EngineType::AMD_GPU) { + const auto t1 = std::chrono::steady_clock::now(); + const double ms = std::chrono::duration(t1 - _t0).count(); + if (ms > kDmlSlowThresholdMs) { + DmlSerialState::TripFromLatency(_context, ms); + } + } + if (_locked) { + DmlSerialState::ReleaseMutex(); + } + } + // ==================================================================== // BasicOrtHandler — constructors // ==================================================================== @@ -365,6 +485,85 @@ namespace ANSCENTER { return false; } + // ==================================================================== + // Reinitialize — tear down + rebuild session on a new EP. + // + // Used by subclass self-heal paths (ONNXOCRRecognizer / + // ONNXOCRDetector) to recover from Ort::Exception in Run(). The + // typical flow is: + // + // try { ort_session->Run(...); } + // catch (const Ort::Exception&) { + // Reinitialize(EngineType::AMD_GPU); // first attempt: retry DML + // try { ort_session->Run(...); } + // catch (const Ort::Exception&) { + // Reinitialize(EngineType::CPU); // last resort: drop to CPU + // ort_session->Run(...); + // } + // } + // + // After this call returns, ort_session, ort_env, memory_info_handler, + // input_node_names, input_node_dims, output_node_names, etc. are all + // freshly populated for `newEngine`. Subclass state that lives outside + // BasicOrtHandler (e.g. ONNXOCRRecognizer::_warmedUp, ::keys_) is NOT + // touched — the caller should reset whatever flags it owns if they + // become invalid for the new session. + // + // Throws Ort::Exception (or std::exception) on failure to construct + // the new session. Caller should treat that as terminal for the + // current Run() and return an empty result. + // ==================================================================== + void BasicOrtHandler::Reinitialize(EngineType newEngine) + { + ANS_DBG("OrtHandler", + "Reinitialize: oldEngine=%d newEngine=%d (model=%ls)", + static_cast(m_engineType), + static_cast(newEngine), + onnx_path ? onnx_path : L"(null)"); + + // Drop the existing session FIRST so its D3D12 / DML allocations + // are released back to the driver. Order matters: ort_session + // must die before ort_env, otherwise Ort::Session's destructor + // can dereference a dangling env pointer in some EP teardown + // paths (observed on DML 1.15.x). + if (ort_session) { + try { delete ort_session; } catch (...) {} + ort_session = nullptr; + } + if (memory_info_handler) { + try { delete memory_info_handler; } catch (...) {} + memory_info_handler = nullptr; + } + if (ort_env) { + try { delete ort_env; } catch (...) {} + ort_env = nullptr; + } + + // Reset shape/name vectors so initialize_handler repopulates + // them from the new session (input names can differ across EPs + // for some models — known DML EP quirk). + input_node_names.clear(); + input_node_names_.clear(); + input_node_dims.clear(); + input_values_handler.clear(); + output_node_names.clear(); + output_node_names_.clear(); + output_node_dims.clear(); + input_name = nullptr; + input_tensor_size = 1; + num_outputs = 1; + + // Set the requested EP and re-run the standard init. Throws on + // failure — caller's responsibility to handle. + m_engineType = newEngine; + initialize_handler(); + + ANS_DBG("OrtHandler", + "Reinitialize OK: ort_session=%p engine=%d", + static_cast(ort_session), + static_cast(m_engineType)); + } + // ==================================================================== // initialize_handler // ==================================================================== diff --git a/engines/ONNXEngine/ONNXEngine.h b/engines/ONNXEngine/ONNXEngine.h index 6e6cb27..1e2d19e 100644 --- a/engines/ONNXEngine/ONNXEngine.h +++ b/engines/ONNXEngine/ONNXEngine.h @@ -8,6 +8,7 @@ #include #include #include +#include // DmlInferenceGuard latency timing #include "onnxruntime_cxx_api.h" #include "opencv2/opencv.hpp" @@ -337,6 +338,108 @@ namespace ANSCENTER { // (AMD fields go here — none yet) }; + // ==================================================================== + // DmlSerialState — process-wide DirectML inference serialization + // + // AMD Radeon iGPUs (Radeon 680M / 780M, observed) cannot sustain + // 2-way concurrent inference on DirectML. Under sustained + // parallel load the WDDM driver eventually deadlocks the D3D12 + // command queue when multiple ORT sessions submit work + // simultaneously, producing a "dual stall" pattern (both threads + // inside ORT::Run() with no exception thrown) and ultimately an + // access violation deep inside the DirectML EP cleanup path. + // + // Policy: every DirectML inference call on AMD_GPU is serialized + // through DmlInferenceGuard's unconditional lock. Lazy "trip on + // first slow Run()" was tried first (a flag flipped to true on + // exception or >1500 ms latency) but field-proven inadequate — + // the trip fires AFTER a slow Run() returns, by which point a + // sibling thread has already entered Run() concurrently and + // wedged. Always-serialize is the only safe answer on AMD. + // + // This class still tracks an "ever-tripped" flag for diagnostics + // — the TripFromException / TripFromLatency calls log a marker + // the first time a DML hiccup is observed, so field traces + // clearly show whether the AMD driver had a degraded event. The + // flag does NOT gate the lock; the lock is unconditional on AMD. + // + // Intel (OpenVINO) and NVIDIA (CUDA/TRT) bypass the lock entirely + // — DmlInferenceGuard short-circuits on ep != AMD_GPU. They keep + // peakDepth=2 parallel inference unchanged; zero behavioral + // change on those platforms. + // + // Storage lives in the ONNXEngine sources compiled into + // ANSODEngine.dll so all higher-level modules (ANSODEngine, + // ANSOCR, ...) share a single process-global mutex. + // ==================================================================== + class ONNXENGINE_API DmlSerialState { + public: + // True when serial mode has been tripped. + static bool IsSerialMode(); + + // Trip helpers. Both are idempotent — only the first caller + // per process actually flips the flag and emits the + // diagnostic. Called by: + // exception path → ANSONNXYOLO::HandleDmlRunFailure, OCR + // latency path → DmlInferenceGuard destructor on slow Run() + static void TripFromException(const char* context, const char* msg); + static void TripFromLatency (const char* context, double ms); + + // 1500 ms — comfortably above normal AMD DML latency and well + // under WDDM TDR. + static double SlowThresholdMs(); + + // Internal — only DmlInferenceGuard should call these. + static void AcquireMutex(); + static void ReleaseMutex(); + }; + + // ==================================================================== + // DmlInferenceGuard — RAII guard for DirectML inference call sites + // + // Construct around every Ort::Session::Run() that may execute on + // a DirectML EP. Behavior: + // + // 1. ep != AMD_GPU → entire guard is a no-op (zero overhead on + // Intel / NVIDIA / CPU). The destructor's latency check and + // lock release are also skipped via _ep == AMD_GPU and + // _locked == false guards. + // 2. ep == AMD_GPU → unconditionally acquires the global DML + // mutex; only one DML inference in the entire process runs + // at a time. On scope exit, releases the mutex and (if the + // call exceeded DmlSerialState::SlowThresholdMs()) records a + // diagnostic trip marker — the marker does NOT change + // behavior, since AMD is already serial; it just shows up in + // logs as evidence of a degraded GPU event. + // + // Why unconditional serialization on AMD: empirical testing on + // Radeon 780M shows the WDDM driver wedges when 2 ORT sessions + // submit DML inference concurrently for >~30 s. Lazy "lock only + // after first slow call" was found insufficient — by the time the + // trip fires, a sibling thread is already inside the wedge-prone + // Run() with no way to back out. Always-lock is the only path + // that guarantees the dual-stall window never opens. The cost is + // ~30% lower throughput on AMD (depth=1 vs depth=2) for + // unbounded uptime instead of crashing every 30 s. + // ==================================================================== + class ONNXENGINE_API DmlInferenceGuard { + public: + explicit DmlInferenceGuard(EngineType ep, const char* context); + ~DmlInferenceGuard(); + + // Non-copyable / non-movable — RAII scope guard only. + DmlInferenceGuard(const DmlInferenceGuard&) = delete; + DmlInferenceGuard& operator=(const DmlInferenceGuard&) = delete; + DmlInferenceGuard(DmlInferenceGuard&&) = delete; + DmlInferenceGuard& operator=(DmlInferenceGuard&&) = delete; + + private: + EngineType _ep; + const char* _context; + bool _locked; + std::chrono::steady_clock::time_point _t0; + }; + // ==================================================================== // BasicOrtHandler // ==================================================================== @@ -410,6 +513,21 @@ namespace ANSCENTER { // (which require the input name) BEFORE the real session is // created. Returns an empty string on failure. static std::string QueryModelInputName(const std::string& onnxPath); + protected: + // Tear down the existing ORT session/env/memory-info and rebuild + // them with `newEngine` as the requested EP. Used by subclass + // self-heal paths (e.g. ONNXOCRRecognizer / ONNXOCRDetector) when + // an Ort::Exception fires from a Run() — the session is recreated + // either on the same EP (transient driver hiccup recovery) or on + // CPU (permanent fallback). Throws on failure; caller should + // treat that as "give up for this call" and return an empty + // result. Subclass-owned auxiliary state (input_node_dims, + // input_node_names, ...) is repopulated by initialize_handler. + // Caller is responsible for any subclass-specific re-init that + // sits on top of the base session (e.g. dictionary loading on + // ONNXOCRRecognizer is not affected because it lives in the + // subclass and survives a session swap). + void Reinitialize(EngineType newEngine); private: void initialize_handler(); protected: diff --git a/modules/ANSCV/ANSFLV.cpp b/modules/ANSCV/ANSFLV.cpp index af8bab0..8346b09 100644 --- a/modules/ANSCV/ANSFLV.cpp +++ b/modules/ANSCV/ANSFLV.cpp @@ -1,6 +1,7 @@ #include "ANSFLV.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include "ANSLicense.h" // ANS_DBG macro #include @@ -785,17 +786,25 @@ extern "C" __declspec(dllexport) int GetFLVCVImage(ANSCENTER::ANSFLVClient** Han // Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock) anscv_mat_replace(image, std::move(img)); - // NV12 GPU fast path (optional — disabled by default for stability) + // NV12 GPU fast path (optional — disabled by default for stability). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. if ((*Handle)->IsNV12FastPath()) { int gpuIdx = (*Handle)->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = (*Handle)->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = (*Handle)->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSFLV V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = (*Handle)->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSFLV V1 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } } @@ -1032,16 +1041,24 @@ extern "C" __declspec(dllexport) int GetFLVCVImage_V2(uint64_t handleVal, int& w } anscv_mat_replace(image, std::move(img)); - // Attach NV12 frame for GPU fast-path inference (side-table registry) + // Attach NV12 frame for GPU fast-path inference (side-table registry). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. int gpuIdx = h->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = h->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = h->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSFLV V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = h->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ h->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSFLV V2 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } diff --git a/modules/ANSCV/ANSFilePlayer.cpp b/modules/ANSCV/ANSFilePlayer.cpp index b390f86..af611e3 100644 --- a/modules/ANSCV/ANSFilePlayer.cpp +++ b/modules/ANSCV/ANSFilePlayer.cpp @@ -1,6 +1,7 @@ #include "ANSFilePlayer.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include #include #include @@ -382,15 +383,24 @@ extern "C" __declspec(dllexport) int GetFilePlayerCVImage(ANSCENTER::ANSFILEPLAY // Attach NV12/CUDA frames to registry for inference zero-copy (same as ANSRTSP/ANSVideoPlayer). // CFilePlayer inherits getNV12Frame()/getCudaHWFrame() from CVideoPlayer. + // AvFramePtr wraps each clone so it auto-frees on any return path before + // the .release() handoff to gpu_frame_attach[_cuda] — leak-by-omission + // becomes impossible if an exception or early return slips in later. int gpuIdx = (*Handle)->_playerClient->getHWDecodingGpuIndex(); - AVFrame* cudaHW = (*Handle)->_playerClient->getCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ (*Handle)->_playerClient->getCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = (*Handle)->_playerClient->getNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ (*Handle)->_playerClient->getNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSFilePlayer V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release()); } else { - AVFrame* nv12 = (*Handle)->_playerClient->getNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->_playerClient->getNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSFilePlayer V1 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } else { // CPU-only fallback: attach full-res BGR clone for ANSRTYOLO cv::Mat infImg = (*Handle)->GetInferenceImage(); @@ -639,16 +649,24 @@ extern "C" __declspec(dllexport) int GetFilePlayerCVImage_V2(uint64_t handleVal, } anscv_mat_replace(image, std::move(img)); - // NV12/CUDA registry attach (same as V1) + // NV12/CUDA registry attach (same as V1). + // AvFramePtr ensures any clone we obtain is freed automatically if we + // don't reach the .release() handoff (e.g. exception, future early return). int gpuIdx = h->_playerClient->getHWDecodingGpuIndex(); - AVFrame* cudaHW = h->_playerClient->getCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ h->_playerClient->getCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = h->_playerClient->getNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ h->_playerClient->getNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSFilePlayer V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release()); } else { - AVFrame* nv12 = h->_playerClient->getNV12Frame(); + ansmedia::AvFramePtr nv12{ h->_playerClient->getNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSFilePlayer V2 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } else { cv::Mat infImg = h->GetInferenceImage(); if (!infImg.empty() && infImg.data != (*image)->data) { diff --git a/modules/ANSCV/ANSMJPEG.cpp b/modules/ANSCV/ANSMJPEG.cpp index 191179a..a6bc807 100644 --- a/modules/ANSCV/ANSMJPEG.cpp +++ b/modules/ANSCV/ANSMJPEG.cpp @@ -1,6 +1,7 @@ #include "ANSMJPEG.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include "ANSLicense.h" // ANS_DBG macro #include @@ -784,17 +785,25 @@ extern "C" __declspec(dllexport) int GetMJPEGCVImage(ANSCENTER::ANSMJPEGClient** // Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock) anscv_mat_replace(image, std::move(img)); - // NV12 GPU fast path (optional — disabled by default for stability) + // NV12 GPU fast path (optional — disabled by default for stability). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. if ((*Handle)->IsNV12FastPath()) { int gpuIdx = (*Handle)->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = (*Handle)->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = (*Handle)->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSMJPEG V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = (*Handle)->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSMJPEG V1 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } } @@ -1030,16 +1039,24 @@ extern "C" __declspec(dllexport) int GetMJPEGCVImage_V2(uint64_t handleVal, int& } anscv_mat_replace(image, std::move(img)); - // Attach NV12 frame for GPU fast-path inference (side-table registry) + // Attach NV12 frame for GPU fast-path inference (side-table registry). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. int gpuIdx = h->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = h->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = h->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSMJPEG V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = h->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ h->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSMJPEG V2 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } diff --git a/modules/ANSCV/ANSRTMP.cpp b/modules/ANSCV/ANSRTMP.cpp index a679bca..80c706e 100644 --- a/modules/ANSCV/ANSRTMP.cpp +++ b/modules/ANSCV/ANSRTMP.cpp @@ -1,6 +1,7 @@ #include "ANSRTMP.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include "ANSLicense.h" // ANS_DBG macro #include @@ -808,17 +809,25 @@ extern "C" __declspec(dllexport) int GetRTMPCVImage(ANSCENTER::ANSRTMPClient** H // Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock) anscv_mat_replace(image, std::move(img)); - // NV12 GPU fast path (optional — disabled by default for stability) + // NV12 GPU fast path (optional — disabled by default for stability). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. if ((*Handle)->IsNV12FastPath()) { int gpuIdx = (*Handle)->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = (*Handle)->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = (*Handle)->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSRTMP V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = (*Handle)->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSRTMP V1 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } } @@ -1053,16 +1062,24 @@ extern "C" __declspec(dllexport) int GetRTMPCVImage_V2(uint64_t handleVal, int& } anscv_mat_replace(image, std::move(img)); - // Attach NV12 frame for GPU fast-path inference (side-table registry) + // Attach NV12 frame for GPU fast-path inference (side-table registry). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. int gpuIdx = h->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = h->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = h->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSRTMP V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = h->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ h->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSRTMP V2 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } diff --git a/modules/ANSCV/ANSRTSP.cpp b/modules/ANSCV/ANSRTSP.cpp index cccdb0f..b89b076 100644 --- a/modules/ANSCV/ANSRTSP.cpp +++ b/modules/ANSCV/ANSRTSP.cpp @@ -2,6 +2,7 @@ #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" #include "GpuNV12SlotPool.h" +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include "ANSLicense.h" // ANS_DBG macro #include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include @@ -1175,7 +1176,12 @@ extern "C" __declspec(dllexport) int GetRTSPCVImage( (void*)*image, gpuIdx, (int)inFlightGuardHeld); if (inFlightGuardHeld) { - AVFrame* cudaHW = (*Handle)->GetCudaHWFrame(); + // AvFramePtr wraps each clone obtained from the player so it + // auto-frees on every return path before the .release() handoff + // to gpu_frame_attach[_cuda]. This closes the leak window if + // any future early-return / exception slips between the clone + // call and the attach call. + ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() }; if (cudaHW) { RTSP_DBG("[GetRTSPCVImage] cudaHW: %dx%d data[0]=%p data[1]=%p", cudaHW->width, cudaHW->height, @@ -1186,13 +1192,20 @@ extern "C" __declspec(dllexport) int GetRTSPCVImage( gpuIdx, cudaHW->width, cudaHW->height); // Only fetch CPU NV12 if pool slot unavailable (cross-GPU fallback). - AVFrame* cpuNV12 = slot ? nullptr : (*Handle)->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12, slot); + ansmedia::AvFramePtr cpuNV12{ slot ? nullptr : (*Handle)->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSRTSP attach_cuda cudaHW=%p cpuNV12=%p slot=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), (void*)slot, gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release(), slot); } else { // HW decode not active — try CPU NV12 - AVFrame* nv12 = (*Handle)->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSRTSP attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } diff --git a/modules/ANSCV/ANSSRT.cpp b/modules/ANSCV/ANSSRT.cpp index af04c0f..7d2c454 100644 --- a/modules/ANSCV/ANSSRT.cpp +++ b/modules/ANSCV/ANSSRT.cpp @@ -1,6 +1,7 @@ #include "ANSSRT.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include "ANSLicense.h" // ANS_DBG macro #include @@ -826,17 +827,25 @@ extern "C" __declspec(dllexport) int GetSRTCVImage(ANSCENTER::ANSSRTClient** Han // Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock) anscv_mat_replace(image, std::move(img)); - // NV12 GPU fast path (optional — disabled by default for stability) + // NV12 GPU fast path (optional — disabled by default for stability). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. if ((*Handle)->IsNV12FastPath()) { int gpuIdx = (*Handle)->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = (*Handle)->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = (*Handle)->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSSRT V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = (*Handle)->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSSRT V1 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } } @@ -1077,16 +1086,24 @@ extern "C" __declspec(dllexport) int GetSRTCVImage_V2(uint64_t handleVal, int& w anscv_mat_replace(image, std::move(img)); - // Attach NV12 frame for GPU fast-path inference (side-table registry) + // Attach NV12 frame for GPU fast-path inference (side-table registry). + // AvFramePtr auto-frees clones if we don't reach the .release() handoff. int gpuIdx = h->GetHWDecodingGpuIndex(); - AVFrame* cudaHW = h->GetCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() }; if (cudaHW) { - AVFrame* cpuNV12 = h->GetNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSSRT V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, + cpuNV12.release()); } else { - AVFrame* nv12 = h->GetNV12Frame(); + ansmedia::AvFramePtr nv12{ h->GetNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSSRT V2 attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } diff --git a/modules/ANSCV/ANSVideoPlayer.cpp b/modules/ANSCV/ANSVideoPlayer.cpp index 70712b7..144392c 100644 --- a/modules/ANSCV/ANSVideoPlayer.cpp +++ b/modules/ANSCV/ANSVideoPlayer.cpp @@ -5,6 +5,7 @@ #include "ANSMatRegistry.h" #include "ANSGpuFrameRegistry.h" #include "ANSGpuFrameOps.h" // gpu_frame_attach, gpu_frame_attach_cuda +#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame* #include #include #include @@ -1010,17 +1011,27 @@ extern "C" __declspec(dllexport) int GetVideoPlayerCVImage( // When using cv::VideoCapture fallback, attach BGR clone (pixFmt=1000). if ((*Handle)->_hwDecodeActive && (*Handle)->_hwPlayer) { // --- HW decode: NV12/CUDA attachment (same pattern as ANSRTSP.cpp:847-862) --- + // AvFramePtr wraps each clone so it auto-frees on any return path + // before the .release() handoff to gpu_frame_attach[_cuda]. This + // makes leak-by-omission impossible if an exception or early return + // is added between the clone and the attach call. int gpuIdx = (*Handle)->_hwPlayer->getHWDecodingGpuIndex(); - AVFrame* cudaHW = (*Handle)->_hwPlayer->getCudaHWFrame(); + ansmedia::AvFramePtr cudaHW{ (*Handle)->_hwPlayer->getCudaHWFrame() }; if (cudaHW) { // CUDA zero-copy: frame data[0]/data[1] are CUDA device pointers. // Also attach CPU NV12 as fallback for cross-GPU inference. - AVFrame* cpuNV12 = (*Handle)->_hwPlayer->getNV12Frame(); - gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12); + ansmedia::AvFramePtr cpuNV12{ (*Handle)->_hwPlayer->getNV12Frame() }; + ANS_DBG("MEDIA_AvPtr", + "ANSVideoPlayer attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d", + (void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx); + gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release()); } else { - AVFrame* nv12 = (*Handle)->_hwPlayer->getNV12Frame(); + ansmedia::AvFramePtr nv12{ (*Handle)->_hwPlayer->getNV12Frame() }; if (nv12) { - gpu_frame_attach(*image, nv12, gpuIdx, timeStamp); + ANS_DBG("MEDIA_AvPtr", + "ANSVideoPlayer attach nv12=%p gpuIdx=%d", + (void*)nv12.get(), gpuIdx); + gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp); } } } else { diff --git a/modules/ANSLPR/ANSLPR.h b/modules/ANSLPR/ANSLPR.h index 8509138..43fed0e 100644 --- a/modules/ANSLPR/ANSLPR.h +++ b/modules/ANSLPR/ANSLPR.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "Utility.h" #include "ANSLicense.h" @@ -98,6 +99,31 @@ namespace ANSCENTER bool _debugFlag{ false }; bool _licenseValid{ false }; bool _isInitialized{ false}; + + // Loading-in-progress flag — distinguishes the "LoadEngine() is still + // running" state from "LoadEngine() never called or failed". Set by + // ModelLoadingGuard at the top of LoadEngine() and cleared on exit + // (success, failure, or exception). RunInference/RunInferencesBatch + // consult this BEFORE _isInitialized so the early frames during the + // 3+ second model-load window don't flood the log with bogus + // "Model is not initialized" errors. + std::atomic _modelLoading{ false }; + + // RAII helper: sets _modelLoading=true on construction, false on + // destruction. Use at the top of LoadEngine() so the flag is always + // cleared on every exit path. + struct ModelLoadingGuard { + std::atomic& flag; + explicit ModelLoadingGuard(std::atomic& f) : flag(f) { + flag.store(true, std::memory_order_release); + } + ~ModelLoadingGuard() { + flag.store(false, std::memory_order_release); + } + ModelLoadingGuard(const ModelLoadingGuard&) = delete; + ModelLoadingGuard& operator=(const ModelLoadingGuard&) = delete; + }; + double _detectorThreshold{ 0.5 }; double _ocrThreshold{ 0.5 }; double _colorThreshold{ 0.0 };// If we use the colour information, this threshold should be greater than 0.0 diff --git a/modules/ANSLPR/ANSLPR_OCR.cpp b/modules/ANSLPR/ANSLPR_OCR.cpp index 0853bfc..29c66d4 100644 --- a/modules/ANSLPR/ANSLPR_OCR.cpp +++ b/modules/ANSLPR/ANSLPR_OCR.cpp @@ -221,6 +221,11 @@ namespace ANSCENTER bool ANSALPR_OCR::LoadEngine() { std::lock_guard lock(_mutex); + // RAII guard: set _modelLoading=true for the lifetime of this scope. + // RunInference / RunInferencesBatch consult this flag and silently + // defer (instead of logging "Model is not initialized") while we're + // loading the LP detector + OCR engine + colour classifier. + ModelLoadingGuard mlg(_modelLoading); try { WriteEventLog("ANSALPR_OCR::LoadEngine: Step 1 - Starting engine load"); this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 1: Starting engine load", __FILE__, __LINE__); @@ -1479,22 +1484,28 @@ namespace ANSCENTER this->_logger.LogError("ANSALPR_OCR::RunInference", "Invalid license", __FILE__, __LINE__); return {}; } - if (!_isInitialized) { - this->_logger.LogError("ANSALPR_OCR::RunInference", "Model is not initialized", __FILE__, __LINE__); + + // LoadEngine() may be running on another thread (3+ seconds for all + // models). Silently defer instead of spamming "Model is not + // initialized" — the caller will retry on the next frame. + if (_modelLoading.load(std::memory_order_acquire)) { return {}; } + + // Source-of-truth check on the actual engine pointers. If LoadEngine + // failed or Destroy() was called, _lpDetector / _ocrEngine are null + // and we want a real error logged so the operator sees it. + if (!_lpDetector || !_ocrEngine) { + this->_logger.LogError("ANSALPR_OCR::RunInference", + "Model not loaded (LoadEngine not called or failed)", + __FILE__, __LINE__); + return {}; + } + if (input.empty() || input.cols < 5 || input.rows < 5) { this->_logger.LogError("ANSALPR_OCR::RunInference", "Input image is empty or too small", __FILE__, __LINE__); return {}; } - if (!_lpDetector) { - this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_lpDetector is null", __FILE__, __LINE__); - return {}; - } - if (!_ocrEngine) { - this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_ocrEngine is null", __FILE__, __LINE__); - return {}; - } try { // Convert grayscale to BGR if necessary @@ -1856,19 +1867,25 @@ namespace ANSCENTER this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", "Invalid license", __FILE__, __LINE__); return {}; } - if (!_isInitialized) { - this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", "Model is not initialized", __FILE__, __LINE__); + + // LoadEngine() may be running on another thread (3+ seconds for all + // models). Silently defer instead of spamming "Model is not + // initialized" — the caller will retry on the next frame. + if (_modelLoading.load(std::memory_order_acquire)) { return {}; } + + // Source-of-truth check on the actual engine pointers. If LoadEngine + // failed or Destroy() was called, _lpDetector / _ocrEngine are null + // and we want a real error logged so the operator sees it. + if (!_lpDetector || !_ocrEngine) { + this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", + "Model not loaded (LoadEngine not called or failed)", + __FILE__, __LINE__); + return {}; + } + if (input.empty() || input.cols < 5 || input.rows < 5) return {}; - if (!_lpDetector) { - this->_logger.LogFatal("ANSALPR_OCR::RunInferencesBatch", "_lpDetector is null", __FILE__, __LINE__); - return {}; - } - if (!_ocrEngine) { - this->_logger.LogFatal("ANSALPR_OCR::RunInferencesBatch", "_ocrEngine is null", __FILE__, __LINE__); - return {}; - } if (vehicleBoxes.empty()) return {}; try { diff --git a/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.cpp b/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.cpp index a8a5c5b..2aaa6f5 100644 --- a/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.cpp +++ b/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.cpp @@ -2,6 +2,7 @@ #include "include/clipper.h" #include "ANSGpuFrameRegistry.h" #include "NV12PreprocessHelper.h" +#include "ANSLicense.h" // ANS_DBG #include #include @@ -12,6 +13,130 @@ namespace ANSCENTER { namespace onnxocr { +// AMD/DirectML latency-based recreate (Path B) — record helper. See +// ONNXOCRRecognizer for the rationale. Arms _slowRecreatePending if +// a Run() exceeded the slow threshold while past warmup and outside +// the cooldown window. +void ONNXOCRDetector::RecordRunLatency(const char* context, double ms) { + if (ms <= kSlowRecreateThresholdMs) return; + if (!_warmedUp) return; + const auto now = std::chrono::steady_clock::now(); + const auto sinceLast = std::chrono::duration_cast( + now - _lastRecreateAt).count(); + if (_lastRecreateAt.time_since_epoch().count() != 0 && + sinceLast < kRecreateCooldownMs) { + ANS_DBG("ANSONNXDet_run", + "%s: slow Run (%.0f ms) but in cooldown (%lldms)", + context, ms, static_cast(sinceLast)); + return; + } + _slowRecreatePending.store(true, std::memory_order_release); + std::cerr << "[ONNXOCRDetector] " << context + << ": slow Run (" << ms + << " ms > threshold) — flagged for proactive recreate\n"; + ANS_DBG("ANSONNXDet_run", + "%s: slow Run %.0fms > %.0fms — armed proactive recreate", + context, ms, kSlowRecreateThresholdMs); +} + +// AMD/DirectML latency-based recreate (Path B) — apply helper. Called +// at entry of public methods. If the slow flag was armed by a prior +// call's RecordRunLatency, swap the session in here so this call uses +// the fresh one. +void ONNXOCRDetector::ApplyPendingSlowRecreate(const char* context) { + if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) { + return; + } + const EngineType currentEp = getEngineType(); + std::cerr << "[ONNXOCRDetector] " << context + << ": proactive recreate on EP=" << static_cast(currentEp) + << " (latency-triggered)\n"; + ANS_DBG("ANSONNXDet_run", + "%s: proactive recreate on EP=%d", context, + static_cast(currentEp)); + try { + Reinitialize(currentEp); + _warmedUp = false; + _lastRecreateAt = std::chrono::steady_clock::now(); + std::cerr << "[ONNXOCRDetector] " << context + << ": proactive recreate OK\n"; + ANS_DBG("ANSONNXDet_run", "%s: proactive recreate OK", context); + } catch (const std::exception& e) { + std::cerr << "[ONNXOCRDetector] " << context + << ": proactive recreate failed: " << e.what() << "\n"; + ANS_DBG("ANSONNXDet_run", + "%s: proactive recreate failed (%s)", context, e.what()); + } catch (...) { + std::cerr << "[ONNXOCRDetector] " << context + << ": proactive recreate failed (unknown)\n"; + } +} + +// AMD/DirectML self-heal helper. See ONNXOCRRecognizer for the full +// rationale. On any Run() failure: recreate session on the same EP +// first (transient driver hiccup recovery — the common case for AMD +// DML 0x80004005 E_FAIL); on a second failure, fall back to CPU EP. +// The current call's result is already lost — caller returns empty; +// the recovered session is used by the NEXT Detect call. +void ONNXOCRDetector::HandleRunFailureAndRecreate(const char* context, + const std::exception& e) { + const EngineType currentEp = getEngineType(); + std::cerr << "[ONNXOCRDetector] " << context + << ": Run() failed on EP=" << static_cast(currentEp) + << ": " << e.what() << " — recreating session\n"; + ANS_DBG("ANSONNXDet_run", + "%s: Run failed EP=%d msg=%.200s — recreating", + context, static_cast(currentEp), e.what()); + + // Option A: trip process-wide DML serial mode on AMD exception + // (idempotent, no-op on Intel/NVIDIA/CPU). + if (currentEp == EngineType::AMD_GPU) { + ANSCENTER::DmlSerialState::TripFromException( + context, e.what()); + } + + if (currentEp != EngineType::CPU) { + try { + Reinitialize(currentEp); + _warmedUp = false; + std::cerr << "[ONNXOCRDetector] " << context + << ": session recreated on EP=" << static_cast(currentEp) << "\n"; + ANS_DBG("ANSONNXDet_run", + "%s: session recreated on EP=%d", context, + static_cast(currentEp)); + return; + } catch (const std::exception& reErr) { + std::cerr << "[ONNXOCRDetector] " << context + << ": same-EP recreate failed: " << reErr.what() + << " — falling back to CPU\n"; + ANS_DBG("ANSONNXDet_run", + "%s: same-EP recreate failed (%s); falling back to CPU", + context, reErr.what()); + } catch (...) { + std::cerr << "[ONNXOCRDetector] " << context + << ": same-EP recreate failed (unknown) — falling back to CPU\n"; + } + } + + try { + Reinitialize(EngineType::CPU); + _warmedUp = false; + std::cerr << "[ONNXOCRDetector] " << context + << ": fallback session created on CPU\n"; + ANS_DBG("ANSONNXDet_run", "%s: CPU fallback OK", context); + } catch (const std::exception& cpuErr) { + std::cerr << "[ONNXOCRDetector] " << context + << ": CPU fallback ALSO failed: " << cpuErr.what() + << " — ort_session may be null, future calls will return empty\n"; + ANS_DBG("ANSONNXDet_run", + "%s: CPU fallback failed (%s); ort_session may be null", + context, cpuErr.what()); + } catch (...) { + std::cerr << "[ONNXOCRDetector] " << context + << ": CPU fallback ALSO failed (unknown)\n"; + } +} + ONNXOCRDetector::ONNXOCRDetector(const std::string& onnx_path, unsigned int num_threads) : BasicOrtHandler(onnx_path, num_threads) { } @@ -55,6 +180,11 @@ std::vector ONNXOCRDetector::Detect(const cv::Mat& srcImage, return {}; } + // Path B (AMD/DML): apply pending proactive recreate before constructing + // any tensors. See ONNXOCRRecognizer for the full rationale. + ApplyPendingSlowRecreate("Detect"); + if (!ort_session) return {}; // recreate may have failed; bail safely + // Try to get full-resolution image from NV12 frame (same pattern as ANSONNXYOLO) cv::Mat inferenceImage = srcImage; float bgrScaleX = 1.0f, bgrScaleY = 1.0f; @@ -89,36 +219,60 @@ std::vector ONNXOCRDetector::Detect(const cv::Mat& srcImage, *memory_info_handler, inputData.data(), inputData.size(), inputShape.data(), inputShape.size()); - // Run inference - auto outputTensors = ort_session->Run( - Ort::RunOptions{ nullptr }, - input_node_names.data(), &inputTensor, 1, - output_node_names.data(), num_outputs); + // Run inference + post-process inside try/catch. AMD/DirectML + // self-heal: on Run() failure (typically 0x80004005 E_FAIL from a + // transient DML driver hiccup) we recreate the session for the next + // call (same EP first, CPU as last resort). This call's result is + // lost — return empty boxes; ALPR pipeline tolerates a dropped + // detection frame, the tracker recovers. Latency is also recorded + // for the Path B proactive-recreate trigger. + try { + const auto t0 = std::chrono::steady_clock::now(); + std::vector outputTensors; + { + // Option A: serialize DML inference once per-process + // serial mode has been tripped (no-op on Intel/NVIDIA/CPU). + ANSCENTER::DmlInferenceGuard dmlGuard( + m_engineType, "ONNXOCRDetector::Detect"); + outputTensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), &inputTensor, 1, + output_node_names.data(), num_outputs); + } + const auto t1 = std::chrono::steady_clock::now(); + const double runMs = + std::chrono::duration(t1 - t0).count(); + RecordRunLatency("Detect", runMs); - // Get output data - float* outputData = outputTensors[0].GetTensorMutableData(); - auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); + // Get output data + float* outputData = outputTensors[0].GetTensorMutableData(); + auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); - int outH = static_cast(outputShape[2]); - int outW = static_cast(outputShape[3]); + int outH = static_cast(outputShape[2]); + int outW = static_cast(outputShape[3]); - // Postprocess — detection coords are relative to inferenceImage (full-res), - // then scaled back to srcImage (display-res) coordinates - auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW, - inferenceImage.rows, inferenceImage.cols, - dbThresh, boxThresh, unclipRatio, useDilation); + // Postprocess — detection coords are relative to inferenceImage (full-res), + // then scaled back to srcImage (display-res) coordinates + auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW, + inferenceImage.rows, inferenceImage.cols, + dbThresh, boxThresh, unclipRatio, useDilation); - // Rescale box coordinates from full-res to display-res - if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) { - for (auto& box : boxes) { - for (auto& pt : box.points) { - pt.x *= bgrScaleX; - pt.y *= bgrScaleY; + // Rescale box coordinates from full-res to display-res + if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) { + for (auto& box : boxes) { + for (auto& pt : box.points) { + pt.x *= bgrScaleX; + pt.y *= bgrScaleY; + } } } - } - return boxes; + return boxes; + } + catch (const Ort::Exception& e) { + HandleRunFailureAndRecreate("Detect", e); + return {}; + } } std::vector ONNXOCRDetector::Preprocess(const cv::Mat& srcImage, int maxSideLen, @@ -323,6 +477,8 @@ void ONNXOCRDetector::Warmup() { // multiple-of-32 rounding. cuDNN caches the algorithm for this // shape so the first real inference doesn't pay the picker cost. constexpr int kWarmupSide = 320; + bool warmupFailed = false; + std::string warmupErr; try { cv::Mat dummy(kWarmupSide, kWarmupSide, CV_8UC3, cv::Scalar(128, 128, 128)); cv::Mat dummyF; @@ -347,6 +503,23 @@ void ONNXOCRDetector::Warmup() { } catch (const Ort::Exception& e) { std::cerr << "[ONNXOCRDetector] Warmup failed: " << e.what() << std::endl; + warmupFailed = true; + warmupErr = e.what(); + } + + // AMD/DML self-heal: if warmup itself threw, the session is dead and + // every Detect() will fail. Recreate now (same EP first, CPU as + // fallback) so production traffic can still serve. Leave _warmedUp + // = false so the recovered session warms up on its first real call. + if (warmupFailed) { + ANS_DBG("ANSONNXDet_run", + "Warmup failed (%.200s); triggering self-heal", + warmupErr.c_str()); + try { + std::runtime_error proxy(warmupErr); + HandleRunFailureAndRecreate("Warmup", proxy); + } catch (...) {} + return; // do NOT set _warmedUp; recovered session warms on first real Detect } _warmedUp = true; } diff --git a/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.h b/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.h index d8620c5..f27debf 100644 --- a/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.h +++ b/modules/ANSOCR/ANSONNXOCR/ONNXOCRDetector.h @@ -4,6 +4,8 @@ #include "ONNXEngine.h" #include #include +#include +#include namespace ANSCENTER { namespace onnxocr { @@ -53,6 +55,25 @@ private: // Expand 4-point box using Clipper offset std::vector UnclipPolygon(const std::array& box, float unclipRatio); + // AMD/DirectML self-heal helper — see ONNXOCRRecognizer for full + // contract. On Run() failure: recreate session on the same EP first + // (transient hiccup recovery), CPU fallback as last resort. Resets + // _warmedUp so the new session gets re-warmed on next Detect call. + void HandleRunFailureAndRecreate(const char* context, const std::exception& e); + + // AMD/DirectML latency-based self-heal (Path B) — see ONNXOCRRecognizer + // for the full rationale. Detector is observed to hit the same + // slow-degradation pattern as the recognizer after a sibling DML + // session takes a fatal: Run() doesn't throw, just gets slow. + // 2 s threshold catches degraded individual Run() (healthy is 50–500 ms); + // 60 s cooldown prevents thrash on permanently broken driver state. + static constexpr double kSlowRecreateThresholdMs = 2000.0; + static constexpr int64_t kRecreateCooldownMs = 60000; + std::atomic _slowRecreatePending{false}; + std::chrono::steady_clock::time_point _lastRecreateAt{}; + void RecordRunLatency(const char* context, double ms); + void ApplyPendingSlowRecreate(const char* context); + std::mutex _mutex; }; diff --git a/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.cpp b/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.cpp index 8cc8a17..ba4a7cb 100644 --- a/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.cpp +++ b/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.cpp @@ -113,6 +113,165 @@ static std::vector PreprocessCropToBucket(const cv::Mat& crop, return padded; } +// AMD/DirectML latency-based recreate (Path B) — record helper. +// +// Called after every successful production-path Run(). If the call +// took longer than kSlowRecreateThresholdMs (5 s default), and the +// session has finished its warmup, and we're outside the cooldown +// window, arm _slowRecreatePending so the NEXT public method will +// perform a session swap before its own Run(). +// +// Doesn't actually do the swap here — that has to happen at a clean +// boundary (no in-flight tensors, no held output Ort::Values), which +// is precisely the start of the next public method. Calling +// Reinitialize from inside the middle of an inference path while +// holding output_node_names / inputTensor / outputTensors references +// would invalidate them under our feet. +void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) { + if (ms <= kSlowRecreateThresholdMs) return; + if (!_warmedUp) return; // first-shape compile is legitimately slow + + // Cooldown gate — if we recreated recently, hold off. Avoids a + // recreate loop if the underlying issue is permanent (e.g. driver + // truly broken) rather than a transient hiccup. + const auto now = std::chrono::steady_clock::now(); + const auto sinceLast = std::chrono::duration_cast( + now - _lastRecreateAt).count(); + if (_lastRecreateAt.time_since_epoch().count() != 0 && + sinceLast < kRecreateCooldownMs) { + ANS_DBG("ANSONNXRec_run", + "%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)", + context, ms, static_cast(sinceLast)); + return; + } + + _slowRecreatePending.store(true, std::memory_order_release); + std::cerr << "[ONNXOCRRecognizer] " << context + << ": slow Run (" << ms + << " ms > threshold) — flagged for proactive session recreate\n"; + ANS_DBG("ANSONNXRec_run", + "%s: slow Run %.0fms > %.0fms — armed proactive recreate", + context, ms, kSlowRecreateThresholdMs); +} + +// AMD/DirectML latency-based recreate (Path B) — apply helper. +// +// Called at the entry of every public method (Recognize, +// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was +// armed by a previous call's RecordRunLatency, do the session swap +// here — before any of this call's tensors are constructed. Caller +// already holds _mutex, so the new ort_session is exclusively ours +// for the rest of this call. +// +// On Reinitialize failure we leave _slowRecreatePending=false so we +// don't loop forever; the existing exception-based HandleRunFailure- +// AndRecreate path will fall back to CPU on the next actual throw. +void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) { + if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) { + return; + } + const EngineType currentEp = getEngineType(); + std::cerr << "[ONNXOCRRecognizer] " << context + << ": proactive recreate on EP=" << static_cast(currentEp) + << " (latency-triggered)\n"; + ANS_DBG("ANSONNXRec_run", + "%s: proactive recreate on EP=%d (latency-triggered)", + context, static_cast(currentEp)); + try { + Reinitialize(currentEp); + _warmedUp = false; + _lastRecreateAt = std::chrono::steady_clock::now(); + std::cerr << "[ONNXOCRRecognizer] " << context + << ": proactive recreate succeeded; next call retries on fresh session\n"; + ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context); + } catch (const std::exception& e) { + std::cerr << "[ONNXOCRRecognizer] " << context + << ": proactive recreate failed: " << e.what() + << " — keeping existing session, exception path will catch real failures\n"; + ANS_DBG("ANSONNXRec_run", + "%s: proactive recreate failed (%s)", context, e.what()); + } catch (...) { + std::cerr << "[ONNXOCRRecognizer] " << context + << ": proactive recreate failed (unknown)\n"; + } +} + +// AMD/DirectML self-heal helper. See header comment for detailed contract. +// In short: on Run() failure, recreate the session on the same EP first +// (transient driver hiccup recovery — the common case); on a second +// failure, fall back to CPU EP. Either path leaves ort_session in a +// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's +// result is already lost — caller returns empty. +void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context, + const std::exception& e) { + const EngineType currentEp = getEngineType(); + std::cerr << "[ONNXOCRRecognizer] " << context + << ": Run() failed on EP=" << static_cast(currentEp) + << ": " << e.what() << " — recreating session\n"; + ANS_DBG("ANSONNXRec_run", + "%s: Run failed EP=%d msg=%.200s — recreating", + context, static_cast(currentEp), e.what()); + + // Option A: a Run() exception on a DML session is the strongest + // signal the AMD driver can't sustain parallel inference. Trip + // process-wide DML serial mode so future inference calls (from + // any module) take the global mutex. Idempotent — only the + // first caller per process flips the flag. No-op on + // Intel/NVIDIA/CPU. + if (currentEp == EngineType::AMD_GPU) { + ANSCENTER::DmlSerialState::TripFromException( + context, e.what()); + } + + // Step 1: try to recover on the same EP. Most DML 0x80004005 failures + // are transient driver hiccups; a fresh session typically works. This + // is also a no-op cost on Intel/NVIDIA — those EPs effectively never + // throw from Run() under normal load, so this path is rarely taken. + if (currentEp != EngineType::CPU) { + try { + Reinitialize(currentEp); + _warmedUp = false; + std::cerr << "[ONNXOCRRecognizer] " << context + << ": session recreated on EP=" << static_cast(currentEp) + << " — next call will retry on GPU\n"; + ANS_DBG("ANSONNXRec_run", + "%s: session recreated on EP=%d (next call retries on GPU)", + context, static_cast(currentEp)); + return; + } catch (const std::exception& reCreateErr) { + std::cerr << "[ONNXOCRRecognizer] " << context + << ": same-EP recreate failed: " << reCreateErr.what() + << " — falling back to CPU\n"; + ANS_DBG("ANSONNXRec_run", + "%s: same-EP recreate failed (%s); falling back to CPU", + context, reCreateErr.what()); + } catch (...) { + std::cerr << "[ONNXOCRRecognizer] " << context + << ": same-EP recreate failed (unknown) — falling back to CPU\n"; + } + } + + // Step 2: last resort — recreate on CPU. Slower but always works + // when the model itself is valid. Caller's next call uses CPU EP. + try { + Reinitialize(EngineType::CPU); + _warmedUp = false; + std::cerr << "[ONNXOCRRecognizer] " << context + << ": fallback session created on CPU\n"; + ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context); + } catch (const std::exception& cpuErr) { + std::cerr << "[ONNXOCRRecognizer] " << context + << ": CPU fallback ALSO failed: " << cpuErr.what() + << " — ort_session likely null, future calls will return empty\n"; + ANS_DBG("ANSONNXRec_run", + "%s: CPU fallback failed (%s); ort_session may be null", + context, cpuErr.what()); + } catch (...) { + std::cerr << "[ONNXOCRRecognizer] " << context + << ": CPU fallback ALSO failed (unknown exception)\n"; + } +} + TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) { std::lock_guard lock(_mutex); @@ -120,6 +279,12 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) { return {}; } + // Path B (AMD/DML): if a previous call observed a slow Run(), it armed + // a proactive-recreate flag. Apply it now — BEFORE this call's tensors + // are built — so the rest of this method runs on a fresh session. + ApplyPendingSlowRecreate("Recognize"); + if (!ort_session) return {}; // recreate could have failed; bail safely + try { // Step 1: aspect-preserving resize to height=imgH_, width capped // at imgMaxW_. Then round resized width up to the next bucket. @@ -133,10 +298,24 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) { *memory_info_handler, inputData.data(), inputData.size(), inputShape.data(), inputShape.size()); - auto outputTensors = ort_session->Run( - Ort::RunOptions{ nullptr }, - input_node_names.data(), &inputTensor, 1, - output_node_names.data(), num_outputs); + const auto t0 = std::chrono::steady_clock::now(); + std::vector outputTensors; + { + // Option A: serialize DML inference once per-process + // serial mode has been tripped (no-op on Intel/NVIDIA/CPU). + ANSCENTER::DmlInferenceGuard dmlGuard( + m_engineType, "ONNXOCRRecognizer::Recognize"); + outputTensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), &inputTensor, 1, + output_node_names.data(), num_outputs); + } + const auto t1 = std::chrono::steady_clock::now(); + const double runMs = + std::chrono::duration(t1 - t0).count(); + // Latency-based recreate (Path B): arms the flag for the NEXT call + // if this one was abnormally slow. + RecordRunLatency("Recognize", runMs); float* outputData = outputTensors[0].GetTensorMutableData(); auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); @@ -147,7 +326,11 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) { return CTCDecode(outputData, seqLen, numClasses); } catch (const Ort::Exception& e) { - std::cerr << "[ONNXOCRRecognizer] Inference failed: " << e.what() << std::endl; + // AMD/DML self-heal: recreate session for next call (same EP first, + // CPU fallback as last resort). This call's output is lost — return + // empty. The OCR pipeline tolerates dropped frames; the ALPR voting + // tracker migrates plate text across frames. + HandleRunFailureAndRecreate("Recognize", e); return {}; } } @@ -158,6 +341,13 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector& crops, std::vector& out) { if (crops.empty()) return; + // Path B (AMD/DML): apply pending proactive recreate before any tensors + // are constructed for this batch. RecognizeBatch holds _mutex around + // every RunBatchAtWidth chunk it dispatches, so we're safe under the + // same lock the caller already owns. + ApplyPendingSlowRecreate("RunBatchAtWidth"); + if (!ort_session) return; // recreate may have failed; abort safely + try { // Real number of crops vs. padded batch size submitted to ORT. // Padding to the ladder keeps the recognizer's input-shape set @@ -189,10 +379,22 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector& crops, *memory_info_handler, batchInput.data(), batchInput.size(), inputShape.data(), inputShape.size()); - auto outputTensors = ort_session->Run( - Ort::RunOptions{ nullptr }, - input_node_names.data(), &inputTensor, 1, - output_node_names.data(), num_outputs); + const auto t0 = std::chrono::steady_clock::now(); + std::vector outputTensors; + { + // Option A: serialize DML inference once per-process + // serial mode has been tripped (no-op on Intel/NVIDIA/CPU). + ANSCENTER::DmlInferenceGuard dmlGuard( + m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth"); + outputTensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), &inputTensor, 1, + output_node_names.data(), num_outputs); + } + const auto t1 = std::chrono::steady_clock::now(); + const double runMs = + std::chrono::duration(t1 - t0).count(); + RecordRunLatency("RunBatchAtWidth", runMs); float* outputData = outputTensors[0].GetTensorMutableData(); auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); @@ -215,11 +417,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector& crops, } } catch (const Ort::Exception& e) { - // ORT will throw if the model doesn't support a batch dimension > 1. - // Fall back to per-image inference for this group. + // The original recovery path here covered "model doesn't support + // batch dim >1" — a one-shot decision: drop to single-image and + // try each crop. We keep that, but ALSO add AMD/DML self-heal + // on top: if the per-image fallback ALSO throws, we recreate the + // session (same EP first, then CPU) so the next caller gets a + // healthy session. The current call's results stay empty for + // any crops we couldn't decode — caller tolerates this. std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW=" << bucketW << " (" << e.what() << ") — falling back to single-image path." << std::endl; + bool anyPerImageFailed = false; + std::string lastPerImageErr; for (size_t i = 0; i < crops.size(); ++i) { // Direct call (we already hold _mutex via the public RecognizeBatch // wrapper). Replicate the single-image preprocessing here to avoid @@ -232,10 +441,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector& crops, Ort::Value inputTensor = Ort::Value::CreateTensor( *memory_info_handler, inputData.data(), inputData.size(), inputShape.data(), inputShape.size()); - auto outputTensors = ort_session->Run( - Ort::RunOptions{ nullptr }, - input_node_names.data(), &inputTensor, 1, - output_node_names.data(), num_outputs); + std::vector outputTensors; + { + // Option A: per-image fallback path also goes + // through the DML serialization gate. + ANSCENTER::DmlInferenceGuard dmlGuard( + m_engineType, + "ONNXOCRRecognizer::RunBatchAtWidth/per-image"); + outputTensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), &inputTensor, 1, + output_node_names.data(), num_outputs); + } float* outData = outputTensors[0].GetTensorMutableData(); auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); int seqLen = static_cast(outShape[1]); @@ -245,6 +462,20 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector& crops, std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: " << e2.what() << std::endl; out[origIndices[i]] = {}; + anyPerImageFailed = true; + lastPerImageErr = e2.what(); + } + } + // If even the per-image fallback path is throwing, the session is + // genuinely unhealthy (DML device-lost, not just a batch-shape bug). + // Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh + // session (same EP first, CPU as last resort). + if (anyPerImageFailed) { + try { + std::runtime_error proxy(lastPerImageErr); + HandleRunFailureAndRecreate("RunBatchAtWidth", proxy); + } catch (...) { + // HandleRunFailureAndRecreate already logs; swallow. } } } @@ -323,10 +554,13 @@ void ONNXOCRRecognizer::Warmup() { // plates pay a one-time spike on first occurrence of batch=16/24. auto totalT0 = std::chrono::high_resolution_clock::now(); int warmedShapes = 0; + int totalShapes = 0; + std::string lastWarmupErr; for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) { const int batchN = kRecWarmupBatchLadder[bs]; for (int b = 0; b < kRecNumBuckets; ++b) { const int bucketW = kRecBucketWidths[b]; + ++totalShapes; try { const size_t perImage = static_cast(3) * imgH_ * bucketW; std::vector batchInput(static_cast(batchN) * perImage, 0.0f); @@ -361,6 +595,7 @@ void ONNXOCRRecognizer::Warmup() { std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch=" << batchN << " bucketW=" << bucketW << ": " << e.what() << std::endl; + lastWarmupErr = e.what(); } } } @@ -368,6 +603,30 @@ void ONNXOCRRecognizer::Warmup() { double totalMs = std::chrono::duration(totalT1 - totalT0).count(); std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes << " shapes in " << totalMs << " ms total" << std::endl; + + // AMD/DML self-heal: if NO shape warmed up, the session is dead and + // production traffic will hit Run() failures on every call. Recreate + // it now (same EP first, CPU as fallback) so the pipeline can still + // serve. We deliberately keep _warmedUp = false in this path; the + // next RecognizeBatch will re-trigger Warmup on the recovered session. + // + // We do NOT trigger self-heal on partial failure — losing 2 of 16 + // shapes is normal (e.g. some bucket dims aren't supported by all + // EPs); the per-shape catch already guards against that. + if (warmedShapes == 0 && totalShapes > 0) { + std::cerr << "[ONNXOCRRecognizer] Warmup: ALL " + << totalShapes << " shapes failed — triggering self-heal\n"; + ANS_DBG("ANSONNXRec_run", + "Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session", + totalShapes, lastWarmupErr.c_str()); + try { + std::runtime_error proxy(lastWarmupErr); + HandleRunFailureAndRecreate("Warmup", proxy); + } catch (...) {} + // Don't set _warmedUp = true; let the recovered session warm up + // itself on the next caller's invocation. + return; + } _warmedUp = true; } diff --git a/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.h b/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.h index 5b3159a..cb88e62 100644 --- a/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.h +++ b/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include namespace ANSCENTER { namespace onnxocr { @@ -51,6 +53,62 @@ private: // CTC greedy decode TextLine CTCDecode(const float* outputData, int seqLen, int numClasses); + // AMD/DirectML self-heal helper. Called by Recognize / RunBatchAtWidth / + // Warmup when ort_session->Run() throws an Ort::Exception. Tries to + // recreate the session on the same EP first (transient driver hiccup + // recovery — the common case for DML 0x80004005 E_FAIL); if that fails, + // falls back to CPU EP. On full failure leaves ort_session in whatever + // state Reinitialize left it (possibly nullptr); the next Recognize + // call will short-circuit on the !ort_session check and return empty. + // Resets _warmedUp so the new session gets re-warmed on first use. + void HandleRunFailureAndRecreate(const char* context, const std::exception& e); + + // ---------------------------------------------------------------- + // AMD/DirectML latency-based self-heal (Path B). + // + // Observed AMD failure mode: after a transient driver event (e.g. + // helmet detector hits DML E_FAIL and CPU-falls-back), DML stays in + // a degraded state. OCR Run() calls do NOT throw, but their latency + // jumps from 50–200 ms to 1000–7000 ms, often rising over time. The + // exception-based self-heal above never fires because there's no + // exception. This proactive path observes the latency and forces a + // session recreate when a Run() exceeds a sanity threshold — the + // fresh session typically returns to baseline latency. + // + // How it works: + // 1. Each public method records its Run() duration. + // 2. If duration > kSlowRecreateThresholdMs AND we're past warmup + // AND we're past the cooldown window, set _slowRecreatePending. + // 3. The NEXT call to a public method checks the flag at entry; if + // set, it calls Reinitialize(currentEp) BEFORE its Run() — so + // the call that sees the flag uses a fresh session and returns + // to fast latency. + // 4. After a recreate, _lastRecreateMs starts the cooldown timer. + // + // Threshold tuning: 2 s catches the AMD-degraded individual Run() + // pattern observed in production (1–2 s per OCR Run() when DML is + // sick). Healthy individual Run() is 50–500 ms, so 2 s leaves + // comfortable headroom. First-shape compiles on OpenVINO can hit + // ~1.7 s but those happen during Warmup, which is gated out below + // via the _warmedUp check. 60 s cooldown prevents thrash if the + // driver is in a permanently bad state — at most one recreate per + // minute per session. + // ---------------------------------------------------------------- + static constexpr double kSlowRecreateThresholdMs = 2000.0; + static constexpr int64_t kRecreateCooldownMs = 60000; + + // Note: a slow Run() detected by RecordRunLatency arms this flag; + // ApplyPendingSlowRecreate (called at every public-method entry) + // performs the actual Reinitialize and clears the flag. std::atomic + // is used so the read at entry is lock-free; the write happens under + // the same _mutex the public methods already hold, so the only race + // we're guarding is "another public method picked up the flag first." + std::atomic _slowRecreatePending{false}; + std::chrono::steady_clock::time_point _lastRecreateAt{}; + + void RecordRunLatency(const char* context, double ms); + void ApplyPendingSlowRecreate(const char* context); + std::vector keys_; int imgH_ = kRecImgH; int imgMaxW_ = kRecImgMaxW; diff --git a/modules/ANSODEngine/ANSCUSTOMDetector.h b/modules/ANSODEngine/ANSCUSTOMDetector.h index 982681e..f9e2a7e 100644 --- a/modules/ANSODEngine/ANSCUSTOMDetector.h +++ b/modules/ANSODEngine/ANSCUSTOMDetector.h @@ -24,7 +24,7 @@ namespace ANSCENTER { std::unique_ptr _customDetector; HMODULE hMod= nullptr; std::string FindANSCustomLibraryName(const std::string& folderPath, const std::string& keyword); - bool CreateCustomDetector(); + bool CreateCustomDetector(); }; } #endif \ No newline at end of file diff --git a/modules/ANSODEngine/ANSEngineCommon.h b/modules/ANSODEngine/ANSEngineCommon.h index 237448c..3f315e1 100644 --- a/modules/ANSODEngine/ANSEngineCommon.h +++ b/modules/ANSODEngine/ANSEngineCommon.h @@ -36,6 +36,57 @@ const int MAX_MISSING_SCREEN = 1000; namespace ANSCENTER { + // ──────────────────────────────────────────────────────────────────────── + // DirectML fatal-error detection + // + // AMD's DirectML EP can fail in several distinct ways that all leave the + // Ort::Session permanently broken and require a fresh CPU-fallback session: + // + // 887A0005 DXGI_ERROR_DEVICE_REMOVED — adapter killed (RDP attach, + // TDR, sleep/wake, driver swap). GetDeviceRemovedReason gives + // the sub-cause but the device is gone either way. + // 80004005 E_FAIL — generic D3D12 / DML failure. Seen from + // DmlGraphFusionHelper::ExecuteOperator and + // DmlCommandRecorder::ExecuteCommandList on AMD when the + // driver couldn't compile/execute a fused subgraph or when + // TDR fires mid-command-list. No structured sub-cause + // available — must treat as device-fatal. + // + // Branding strings ("DmlFusedNode", "DmlGraphFusionHelper", + // "DmlCommandRecorder") catch future failure modes that surface the same + // way without us having to chase each new HRESULT. + // + // Used by every catch site that wraps Ort::Session::Run on AMD DML. + // Centralised here so a new failure mode is handled once, not 4+ times. + // ──────────────────────────────────────────────────────────────────────── + inline bool IsDmlFatalError(const std::string& msg) { + return msg.find("887A0005") != std::string::npos + || msg.find("80004005") != std::string::npos + || msg.find("DmlFusedNode") != std::string::npos + || msg.find("DmlGraphFusionHelper") != std::string::npos + || msg.find("DmlCommandRecorder") != std::string::npos + || msg.find("DmlExecutionProvider") != std::string::npos; + } + + // ──────────────────────────────────────────────────────────────────────── + // IsDmlDeviceRemoved — narrowly checks ONLY 887A0005 + // (DXGI_ERROR_DEVICE_REMOVED). This is the truly unrecoverable case: + // the D3D12 adapter is gone (driver crashed, RDP attach killed the + // device, sleep/wake torched the WDDM context). No amount of session + // recreation will bring it back — only a process restart or CPU + // fallback can recover. + // + // In contrast, 80004005 (E_FAIL) from DmlGraphFusionHelper / + // DmlCommandRecorder is intermittent on AMD when the driver chokes on + // a fused subgraph or TDR fires mid-command-list. A fresh DML + // session typically clears it and resumes GPU inference — that's the + // stay-on-GPU recovery path used by ANSONNXYOLO's catch blocks. + // ──────────────────────────────────────────────────────────────────────── + inline bool IsDmlDeviceRemoved(const std::string& msg) { + return msg.find("887A0005") != std::string::npos + || msg.find("DXGI_ERROR_DEVICE_REMOVED") != std::string::npos; + } + template typename std::enable_if::value, T>::type inline clamp(const T& value, const T& low, const T& high) diff --git a/modules/ANSODEngine/ANSONNXCL.cpp b/modules/ANSODEngine/ANSONNXCL.cpp index e23eff9..1ffb5e6 100644 --- a/modules/ANSODEngine/ANSONNXCL.cpp +++ b/modules/ANSODEngine/ANSONNXCL.cpp @@ -1,4 +1,5 @@ #include"ANSONNXCL.h" +#include "ANSONNXYOLO.h" // ANSONNXYOLO is the delegate target — see facade comment in ANSONNXCL.h #include "EPLoader.h" #include "OpenVINODeviceConfig.h" #include // typeid(e).name() in ANS_DBG diagnostics @@ -876,9 +877,50 @@ namespace ANSCENTER ); } catch (const Ort::Exception& e) { + const std::string msg = e.what(); this->_logger.LogError("ANSONNXCL::classify", - "ONNX Runtime Exception during Run(): " + std::string(e.what()), __FILE__, __LINE__); + "ONNX Runtime Exception during Run(): " + msg, __FILE__, __LINE__); delete[] blobPtr; // Clean up on error + blobPtr = nullptr; + + // DML device-lost / DML graph-execution failure → tear down the + // dead DirectML session and rebuild on CPU. Without this, every + // subsequent classify() call hits the same dead session and + // floods the event log (16k+ entries observed on AMD/RDP). + if (IsDmlFatalError(msg) && !_dmlDeviceLost) { + _dmlDeviceLost = true; + _logger.LogFatal("ANSONNXCL::classify", + std::string("DirectML fatal error — recreating session on CPU. msg=") + + msg.substr(0, 256), + __FILE__, __LINE__); + ANS_DBG("ANSONNXCL", "DML fatal — rebuild on CPU msg=%s", msg.c_str()); + try { + // Order matters: drop AllocatedStringPtrs (which depend + // on the live session's allocator) BEFORE tearing down + // session_, sessionOptions_, env_. Init() will repopulate. + inputNodeNameAllocatedStrings_.clear(); + outputNodeNameAllocatedStrings_.clear(); + inputNames_.clear(); + outputNames_.clear(); + session_ = Ort::Session{ nullptr }; + sessionOptions_ = Ort::SessionOptions{ nullptr }; + env_ = Ort::Env{ nullptr }; + if (Init(_modelFilePath, inputImageShape_, /*useGPU=*/false)) { + _logger.LogInfo("ANSONNXCL::classify", + "CPU fallback session created successfully", + __FILE__, __LINE__); + ANS_DBG("ANSONNXCL", "CPU fallback OK"); + } else { + _logger.LogFatal("ANSONNXCL::classify", + "CPU fallback session creation failed", + __FILE__, __LINE__); + } + } catch (const std::exception& re) { + _logger.LogFatal("ANSONNXCL::classify", + std::string("CPU fallback exception: ") + re.what(), + __FILE__, __LINE__); + } + } return {}; } @@ -922,216 +964,90 @@ namespace ANSCENTER return {}; } } - bool ANSONNXCL::OptimizeModel(bool fp16, std::string & optimizedModelFolder) { - if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) { - return false; - } - return true; - } - bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) { - std::lock_guard lock(_mutex); - ModelLoadingGuard mlg(_modelLoading); - try { - _modelLoadValid = false; - bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap); - if (!result) return false; - // Parsing for YOLO only here - _modelConfig = modelConfig; - _modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION; - _modelConfig.modelType = ModelType::ONNXCL; - _modelConfig.inpHeight = 224; - _modelConfig.inpWidth = 224; - if (_modelConfig.modelMNSThreshold < 0.2) - _modelConfig.modelMNSThreshold = 0.5; - if (_modelConfig.modelConfThreshold < 0.2) - _modelConfig.modelConfThreshold = 0.5; - if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max - _modelConfig.numKPS = 17; - if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define - _fp16 = (modelConfig.precisionType == PrecisionType::FP16); - - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1]; - } + // ──────────────────────────────────────────────────────────────────────── + // Public-method facade: delegate everything to the wrapped ANSONNXYOLO. + // + // This is a verification harness — ANSONNXYOLO already covers the + // classification path (auto-detected from model output shape) and ships + // with the safe DML config + DML-fatal-error recovery. By routing every + // ANSONNXCL public call through ANSONNXYOLO we can A/B test whether + // ANSONNXCL's own session/postprocess logic is the actual cause of the + // AMD same-process DML races we saw with the custom helmet model. + // + // The dead code below the facade (Init / preprocess / postprocess / + // classify / Ort::* members in the header) is intentionally preserved + // so the original implementation is one git-revert away if needed. + // ──────────────────────────────────────────────────────────────────────── + namespace { + // Lazy-create the delegate on first public call. Not thread-safe in + // the strict sense, but the public methods all take _mutex first via + // the underlying ANSODBase, so concurrent first-time creation is + // serialised in practice. + inline ANSONNXYOLO& GetDelegate(std::unique_ptr& slot) { + if (!slot) { + slot = std::make_unique(); + ANS_DBG("ANSONNXCL_Facade", + "delegate created (ANSONNXYOLO=%p)", (void*)slot.get()); } - else {// This is old version of model zip file - _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); - _classFilePath = CreateFilePath(_modelFolder, "classes.names"); - std::ifstream isValidFileName(_classFilePath); - if (!isValidFileName) - { - this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__); - LoadClassesFromString(); - } - else { - this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__); - LoadClassesFromFile(); - } - } - // 1. Load labelMap and engine - labelMap.clear(); - if (!_classes.empty()) - labelMap = VectorToCommaSeparatedString(_classes); - - // 2. Initialize ONNX Runtime session - cv::Size targetInputShape= cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight); - result = Init(_modelFilePath, targetInputShape,true); - _modelLoadValid = true; - _isInitialized = true; - return result; - } - catch (const std::exception& e) { - this->_logger.LogFatal("ANSONNXCL::Initialize", e.what(), __FILE__, __LINE__); - return false; + return *slot; } } - bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) { - std::lock_guard lock(_mutex); - ModelLoadingGuard mlg(_modelLoading); - try { - bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword); - if (!result) return false; - _modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION; - _modelConfig.modelType = ModelType::TENSORRT; - _modelConfig.inpHeight = 224; - _modelConfig.inpWidth = 224; - if (_modelConfig.modelMNSThreshold < 0.2) - _modelConfig.modelMNSThreshold = 0.5; - if (_modelConfig.modelConfThreshold < 0.2) - _modelConfig.modelConfThreshold = 0.5; - if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max - _modelConfig.numKPS = 17; - if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define - // if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true; - _fp16 = true; // Load Model from Here - // 0. Check if the configuration file exist - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1]; - } - } - else {// This is old version of model zip file - _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); - _classFilePath = CreateFilePath(_modelFolder, "classes.names"); - std::ifstream isValidFileName(_classFilePath); - if (!isValidFileName) - { - this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__); - LoadClassesFromString(); - } - else { - this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__); - LoadClassesFromFile(); - } - } - // Initialize ONNX Runtime session - cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight); - result = Init(_modelFilePath, targetInputShape,true); - _modelLoadValid = true; - _isInitialized = true; - return result; - } - catch (const std::exception& e) { - this->_logger.LogFatal("ANSONNXCL::LoadModel", e.what(), __FILE__, __LINE__); - return false; - } + bool ANSONNXCL::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { + ANS_DBG("ANSONNXCL_Facade", "OptimizeModel → ANSONNXYOLO"); + return GetDelegate(_yoloDelegate).OptimizeModel(fp16, optimizedModelFolder); } - bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) { - std::lock_guard lock(_mutex); - ModelLoadingGuard mlg(_modelLoading); - try { - bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap); - if (!result) return false; - std::string _modelName = modelName; - if (_modelName.empty()) { - _modelName = "train_last"; - } - std::string modelFullName = _modelName + ".onnx"; - // Parsing for YOLO only here - _modelConfig = modelConfig; - _modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION; - _modelConfig.modelType = ModelType::TENSORRT; - _modelConfig.inpHeight = 224; - _modelConfig.inpWidth = 224; - if (_modelConfig.modelMNSThreshold < 0.2) - _modelConfig.modelMNSThreshold = 0.5; - if (_modelConfig.modelConfThreshold < 0.2) - _modelConfig.modelConfThreshold = 0.5; - if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max - _modelConfig.numKPS = 17; - if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define - _fp16 = true; // Load Model from Here - // 0. Check if the configuration file exist - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1]; - } - } - else {// This is old version of model zip file - _modelFilePath = CreateFilePath(_modelFolder, modelFullName); - _classFilePath = CreateFilePath(_modelFolder, className); - std::ifstream isValidFileName(_classFilePath); - if (!isValidFileName) - { - this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__); - LoadClassesFromString(); - } - else { - this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__); - LoadClassesFromFile(); - } - } - // 1. Load labelMap and engine - labelMap.clear(); - if (!_classes.empty()) - labelMap = VectorToCommaSeparatedString(_classes); - // 2. Initialize ONNX Runtime session - cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight); - result = Init(_modelFilePath, targetInputShape,true); - _modelLoadValid = true; - _isInitialized = true; - return result; - } - catch (const std::exception& e) { - this->_logger.LogFatal("ANSONNXCL::LoadModelFromFolder", e.what(), __FILE__, __LINE__); - return false; - } + bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig, + const std::string& modelZipFilePath, + const std::string& modelZipPassword, + std::string& labelMap) { + ANS_DBG("ANSONNXCL_Facade", "Initialize → ANSONNXYOLO zip=%s", + modelZipFilePath.c_str()); + return GetDelegate(_yoloDelegate).Initialize( + licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap); } + + bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath, + const std::string& modelZipPassword) { + ANS_DBG("ANSONNXCL_Facade", "LoadModel → ANSONNXYOLO zip=%s", + modelZipFilePath.c_str()); + return GetDelegate(_yoloDelegate).LoadModel(modelZipFilePath, modelZipPassword); + } + + bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, + std::string modelName, std::string className, + const std::string& modelFolder, + std::string& labelMap) { + ANS_DBG("ANSONNXCL_Facade", + "LoadModelFromFolder → ANSONNXYOLO modelName=%s folder=%s", + modelName.c_str(), modelFolder.c_str()); + return GetDelegate(_yoloDelegate).LoadModelFromFolder( + licenseKey, modelConfig, modelName, className, modelFolder, labelMap); + } + bool ANSONNXCL::Destroy() { - return true; + if (_yoloDelegate) { + ANS_DBG("ANSONNXCL_Facade", "Destroy → ANSONNXYOLO"); + return _yoloDelegate->Destroy(); + } + return true; } + ANSONNXCL::~ANSONNXCL() { + // Explicit Destroy() so the delegate's session is torn down before + // the unique_ptr destructor frees the object. unique_ptr + // still releases the heap memory afterwards. Destroy(); } - std::vector ANSONNXCL::RunInference(const cv::Mat& input, const std::string& camera_id) { - if (!PreInferenceCheck("ANSONNXCL::RunInference")) return {}; - try { - std::vector result; - if (input.empty()) return result; - if ((input.cols < 5) || (input.rows < 5)) return result; - return classify(input, camera_id); - } - catch (const std::exception& e) { - this->_logger.LogFatal("ANSONNXCL::RunInference", e.what(), __FILE__, __LINE__); - return {}; - } + std::vector ANSONNXCL::RunInference(const cv::Mat& input, + const std::string& camera_id) { + if (input.empty()) return {}; + if (input.cols < 5 || input.rows < 5) return {}; + return GetDelegate(_yoloDelegate).RunInference(input, camera_id); } + std::vector ANSONNXCL::RunInference(const cv::Mat& inputImgBGR) { return RunInference(inputImgBGR, "CustomCam"); } diff --git a/modules/ANSODEngine/ANSONNXCL.h b/modules/ANSODEngine/ANSONNXCL.h index 200a18d..360ae22 100644 --- a/modules/ANSODEngine/ANSONNXCL.h +++ b/modules/ANSODEngine/ANSONNXCL.h @@ -3,8 +3,26 @@ #pragma once #include "ANSEngineCommon.h" #include "engine.h" +#include namespace ANSCENTER { + // Forward declaration: avoid pulling ANSONNXYOLO.h into every consumer of + // ANSONNXCL.h. Definition is included in ANSONNXCL.cpp. + class ANSONNXYOLO; + + // ──────────────────────────────────────────────────────────────────────── + // ANSONNXCL — verification facade over ANSONNXYOLO. + // + // ANSONNXYOLO already auto-detects classification models (single 2D + // output [B, nc]) and routes through its own classification postprocess. + // It also has the safe DML config (DisableMemPattern + ORT_SEQUENTIAL + + // EXTENDED) and the IsDmlFatalError catch-and-rebuild path. + // + // We delegate every public method here to an internal ANSONNXYOLO so we + // can A/B test whether ANSONNXCL's own session/postprocess code is the + // real cause of the AMD same-process DML races we saw with the custom + // helmet model. Public ABI is unchanged — external callers don't move. + // ──────────────────────────────────────────────────────────────────────── class ANSENGINE_API ANSONNXCL :public ANSODBase { public: virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override; @@ -15,10 +33,23 @@ namespace ANSCENTER { std::vector RunInference(const cv::Mat& input, const std::string& camera_id); bool Destroy(); ~ANSONNXCL(); + private: + // The wrapped engine. Lazy-initialised on first public call so we + // don't pay the cost (and pull in the dependency) for callers who + // never use this class. + std::unique_ptr _yoloDelegate; private: std::string _modelFilePath; bool _modelLoadValid; bool _fp16{ false }; + + // DML device-lost recovery: when DirectML fails fatally + // (HRESULT 887A0005 device-removed, 80004005 graph/command-recorder + // E_FAIL, etc. — see ANSCENTER::IsDmlFatalError) the Ort::Session + // is permanently broken. We detect once, recreate on CPU via + // Init(_modelFilePath, inputImageShape_, /*useGPU=*/false), and + // suppress further error-log flooding via this flag. + bool _dmlDeviceLost{ false }; size_t vectorProduct(const std::vector& vector); bool preprocessImageToTensor(const cv::Mat& image, cv::Mat& outImage, const cv::Size& targetShape, diff --git a/modules/ANSODEngine/ANSONNXYOLO.cpp b/modules/ANSODEngine/ANSONNXYOLO.cpp index 47d2294..5171a82 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.cpp +++ b/modules/ANSODEngine/ANSONNXYOLO.cpp @@ -1610,7 +1610,7 @@ namespace ANSCENTER { "Model file does not exist: " + _modelFilePath, __FILE__, __LINE__); return false; } - m_ortEngine = std::make_unique(_modelFilePath); + m_ortEngine = std::make_shared(_modelFilePath); return true; } catch (const std::exception& e) { @@ -1626,7 +1626,7 @@ namespace ANSCENTER { "Model file does not exist: " + _modelFilePath, __FILE__, __LINE__); return false; } - m_ortEngine = std::make_unique(_modelFilePath, engineType); + m_ortEngine = std::make_shared(_modelFilePath, engineType); return true; } catch (const std::exception& e) { @@ -1638,26 +1638,35 @@ namespace ANSCENTER { // ======================================================================== // WarmUpEngine — run a dummy inference after session creation. // - // Scope: **NVIDIA (CUDA EP) only.** On first inference, the CUDA EP - // allocates its memory arena (capped at 2 GB via BasicOrtHandler config), - // resolves cuDNN convolution algorithms, and populates the kernel launch - // cache. Running one dummy inference at load time amortises this cost - // so the first real frame doesn't see a latency spike. + // Scope: NVIDIA (CUDA EP) AND AMD (DirectML EP). + // • NVIDIA: amortises CUDA arena allocation, cuDNN convolution algo + // selection, and kernel-launch cache population. + // • AMD: amortises D3D12 PSO compilation and first-time DML kernel + // binding. Empirically ~150 ms on DirectML (see comment in + // ONNXOCRRecognizer::Warmup). Without this, that compile cost + // surfaces inside the first live Run() — which on AMD has been + // observed to race with concurrent DML traffic from other sessions + // (e.g. helmet detector running while ALPR's _lpDetector fires its + // first ever Run()) and trigger DmlGraphFusionHelper crashes / + // 887A0006 DEVICE_HUNG. Moving the heavy first-run work into + // LoadEngine — where only the loading thread is active — sidesteps + // this timing window. // - // Explicitly disabled on AMD, Intel and CPU: - // • AMD (DirectML) — calling detect() at load time has been observed - // to hit a multi-terabyte cv::Mat allocation inside postprocessLegacy - // on AMD RDNA iGPUs when DirectML returns garbage output tensor - // dims. ONNXYOLO::detect() now has an output-shape sanity guard - // that catches this at runtime, so the warm-up would add risk - // without benefit. Earlier builds enabled warm-up specifically for - // Radeon 680M TDR mitigation; that workaround is obsolete with - // current DirectML 1.15.x drivers. - // • Intel (OpenVINO) — running detect() at load time has been - // observed to expose latent heap-corruption bugs - // (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374). - // • CPU EP — no shader compile or kernel cache to warm up; the first - // real frame has the same latency as any subsequent frame. + // Skipped for: + // • Intel (OpenVINO EP) — OpenVINO compiles the full plan at session + // creation, so first Run() has no extra compile cost. Calling + // detect() at load time has also exposed latent heap-corruption + // bugs (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374). + // • CPU EP — no shader / kernel cache to warm up. + // + // History note: AMD warm-up was disabled in earlier builds after + // observing multi-terabyte cv::Mat allocations from garbage DirectML + // output dims. ONNXYOLO::detect() now has an output-shape sanity + // guard (see line ~257, kMaxOutputDim = 1,000,000) that catches that + // case before postprocessLegacy can allocate, so re-enabling AMD + // warm-up is safe. The exception-swallowing catch blocks below + // guarantee that any warm-up failure is non-fatal — the session is + // already loaded and real inference will still work. // // Non-fatal on failure: if warm-up itself throws, regular inference // still works — the engine is fully loaded before WarmUpEngine runs. @@ -1665,9 +1674,11 @@ namespace ANSCENTER { void ANSONNXYOLO::WarmUpEngine() { if (!m_ortEngine) return; - // Gate strictly on NVIDIA_GPU. Every other EP is a no-op. - if (m_ortEngine->getEngineType() != EngineType::NVIDIA_GPU) { - ANS_DBG("ONNXYOLO", "Warm-up skipped (non-NVIDIA EP)"); + // Warm up on NVIDIA and AMD (both have lazy first-Run() compile cost). + // Skip on CPU and OpenVINO — see scope comment above. + const auto et = m_ortEngine->getEngineType(); + if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) { + ANS_DBG("ONNXYOLO", "Warm-up skipped (CPU/OpenVINO EP)"); return; } @@ -1696,7 +1707,8 @@ namespace ANSCENTER { // preprocessing (114,114,114 ~ 128) and avoids degenerate inputs. cv::Mat dummy(h, w, CV_8UC3, cv::Scalar(128, 128, 128)); - ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy CUDA inference (%dx%d)", w, h); + ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy %s inference (%dx%d)", + (et == EngineType::NVIDIA_GPU) ? "CUDA" : "DirectML", w, h); auto t0 = std::chrono::steady_clock::now(); (void)m_ortEngine->detect(dummy, _classes, @@ -1975,6 +1987,150 @@ namespace ANSCENTER { } } + // ==================================================================== + // HandleDmlRunFailure — stay-on-DML recovery (replaces CPU-fallback). + // + // Called from every catch block that wraps an Ort::Session::Run on a + // DirectML EP. The contract is documented on the member declaration + // in ANSONNXYOLO.h — quick summary: + // + // * CAS on _dmlRecreateInProgress elects ONE recreate owner per + // driver hiccup; other concurrent failures just bail. + // * Cooldown (kDmlRecreateCooldownMs) prevents thrashing if E_FAIL + // keeps firing back to back. + // * IsDmlDeviceRemoved (887A0005) → CPU fallback, last resort. + // * After kMaxConsecutiveDmlFailures recreates without an + // intervening successful inference → CPU fallback. + // * Otherwise: reset() + InitOrtEngine() + + // WarmUpEngine() → fresh DML session, GPU inference resumes. + // + // Successful inferences reset _dmlConsecutiveFailures to 0 in + // DetectObjects / DetectObjectsBatch, so isolated transient hiccups + // do not accumulate toward the CPU-fallback threshold. + // ==================================================================== + void ANSONNXYOLO::HandleDmlRunFailure(const char* context, const std::string& msg) + { + // Option A: a DML exception is the strongest possible signal + // that the AMD driver can't sustain parallel inference. Trip + // process-wide DML serial mode immediately so the NEXT + // inference attempt (from any thread, in this DLL or another) + // takes the global mutex and runs one-at-a-time. This is + // idempotent — only the first caller per process actually + // flips the flag. Done before the CAS so even the CAS losers + // (concurrent threads catching the same fatal) contribute the + // signal — but in practice the first one wins. + ANSCENTER::DmlSerialState::TripFromException(context, msg.c_str()); + + // CAS — only one thread owns the recreate at a time. Other + // concurrent DML failures just return and retry on the next call. + bool expected = false; + if (!_dmlRecreateInProgress.compare_exchange_strong(expected, true)) { + ANS_DBG("ONNXYOLO", "%s: DML fatal — another thread is recreating, skip", + context); + return; + } + + // Hold the engine mutex for the entire recreate so concurrent + // inference threads (which use a shared_ptr local copy of the + // engine) don't see a half-built session. + std::lock_guard lk(_mutex); + + try { + // Cooldown — if we just recreated within the window, the + // driver is in a prolonged bad state; don't burn time + // recreating again immediately. Other threads will keep + // returning {} until either the next call past cooldown + // succeeds or we hit the CPU-fallback threshold. + const auto now = std::chrono::steady_clock::now(); + const auto sinceLastMs = std::chrono::duration_cast( + now - _lastDmlRecreateAt).count(); + if (_lastDmlRecreateAt.time_since_epoch().count() != 0 + && sinceLastMs < kDmlRecreateCooldownMs) { + ANS_DBG("ONNXYOLO", + "%s: DML fatal — within cooldown (%lld ms < %lld), skip", + context, (long long)sinceLastMs, (long long)kDmlRecreateCooldownMs); + _dmlRecreateInProgress.store(false); + return; + } + _lastDmlRecreateAt = now; + + // Decide target EP for the new session: + // * 887A0005 device-removed → CPU is the only option + // * Too many consecutive failures → CPU as last resort + // * Otherwise → recreate on the SAME DML EP we were on + ANSCENTER::EngineType originalEp = ANSCENTER::EngineType::CPU; + if (m_ortEngine) { + originalEp = m_ortEngine->getEngineType(); + } + + const int failures = _dmlConsecutiveFailures.fetch_add(1) + 1; + const bool deviceRemoved = ANSCENTER::IsDmlDeviceRemoved(msg); + const bool exceededRetries = failures > kMaxConsecutiveDmlFailures; + + ANSCENTER::EngineType targetEp = originalEp; + const char* reason = "stay-on-DML retry"; + if (deviceRemoved) { + targetEp = ANSCENTER::EngineType::CPU; + reason = "device-removed (887A0005) → CPU fallback"; + } else if (exceededRetries) { + targetEp = ANSCENTER::EngineType::CPU; + reason = "exceeded retry budget → CPU fallback"; + } + + _logger.LogFatal(context, + std::string("DirectML fatal — ") + reason + + " (failure #" + std::to_string(failures) + + "). msg=" + msg.substr(0, 256), + __FILE__, __LINE__); + ANS_DBG("ONNXYOLO", + "%s: DML fatal recreate — %s, failures=%d, originalEp=%d, targetEp=%d", + context, reason, failures, (int)originalEp, (int)targetEp); + + // reset() releases the member's reference; concurrent inference + // threads holding a shared_ptr local copy keep their old + // engine alive until they return — no use-after-free (B2). + m_ortEngine.reset(); + if (!InitOrtEngine(targetEp)) { + _logger.LogFatal(context, + "DML recreate failed — m_ortEngine left null", __FILE__, __LINE__); + ANS_DBG("ONNXYOLO", "%s: InitOrtEngine(%d) failed", context, (int)targetEp); + _dmlRecreateInProgress.store(false); + return; + } + + // Warm up the new session so the first real call doesn't pay + // the algorithm-selection / shader-compile tax. AMD DML + // recreate WITHOUT warmup typically hits 1-2 s on the very + // first inference, which can re-trigger our 2 s slow watchdog + // and produce a false alarm. + try { + WarmUpEngine(); + } catch (const std::exception& we) { + ANS_DBG("ONNXYOLO", "%s: WarmUpEngine after recreate threw: %s", + context, we.what()); + } + + // If we landed on CPU as a last resort, leave the failure + // counter pinned so we don't oscillate back to DML on the + // next exception. Otherwise (we're back on DML) DON'T reset + // the counter here — it gets reset by a *successful* + // inference in DetectObjects / DetectObjectsBatch. + if (targetEp == ANSCENTER::EngineType::CPU + && originalEp != ANSCENTER::EngineType::CPU) { + _logger.LogInfo(context, + "Permanently switched to CPU EP — DML retry budget exhausted " + "or device removed. Restart the process to retry GPU.", + __FILE__, __LINE__); + } + } + catch (const std::exception& re) { + _logger.LogFatal(context, + std::string("DML recreate exception: ") + re.what(), + __FILE__, __LINE__); + } + _dmlRecreateInProgress.store(false); + } + std::vector ANSONNXYOLO::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) { @@ -1985,16 +2141,27 @@ namespace ANSCENTER { return {}; } - // ── Snapshot config under a brief lock ────────────────── + // ── Snapshot config + engine handle under a brief lock ── // Only hold _mutex long enough to validate state and copy // the parameters needed for inference. The actual ORT // detect() call runs outside the lock so that concurrent // Initialize/LoadModel calls are not blocked for the full // duration of inference. + // + // B2: We also snapshot a shared_ptr COPY of m_ortEngine + // here. The lock-free detect() call below dereferences + // the LOCAL copy (`engine`), not the member. If a + // concurrent thread catches a DML fatal and does + // m_ortEngine.reset() under the same lock, that only + // releases the member's reference — our local copy keeps + // the engine alive until this function returns. This + // closes the use-after-free race that produced the + // 0xc0000005 crash at ANSODEngine.dll +0x111fd0. float probThresh, nmsThresh; int numKps; std::vector classes; bool trackerEnabled, stabilizationEnabled; + std::shared_ptr engine; { ANS_DBG("ONNXYOLO", "DetectObjects: cam=%s acquiring mutex...", camera_id.c_str()); auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjects"); @@ -2007,13 +2174,14 @@ namespace ANSCENTER { return {}; } - // Snapshot parameters while locked + // Snapshot parameters + engine handle while locked probThresh = PROBABILITY_THRESHOLD; nmsThresh = NMS_THRESHOLD; numKps = NUM_KPS; classes = _classes; trackerEnabled = _trackerEnabled; stabilizationEnabled = _stabilizationEnabled; + engine = m_ortEngine; // ref-count++ } // ── _mutex released — heavy work below runs lock-free ─── @@ -2047,11 +2215,28 @@ namespace ANSCENTER { } } - // Run ORT inference — no mutex held, this is the expensive call - auto results = m_ortEngine->detect(inferenceImage, classes, - probThresh, - nmsThresh, - numKps); + // Run ORT inference — no engine mutex held, this is the + // expensive call. + // B2: dereference the LOCAL shared_ptr `engine`, not the + // member m_ortEngine. A concurrent recreate thread may + // reset() m_ortEngine while we're inside detect() — the + // local ref-count keeps the underlying ONNXYOLO alive + // until this function returns. + // + // Option A: DmlInferenceGuard wraps the call to (a) + // serialize on the global DML mutex IF DML serial mode has + // been tripped, and (b) trip serial mode if THIS call's + // latency exceeds the slow threshold (1.5 s — well below + // WDDM TDR). No-op on Intel/NVIDIA/CPU. + std::vector results; + { + DmlInferenceGuard dmlGuard(engine->getEngineType(), + "ANSONNXYOLO::DetectObjects"); + results = engine->detect(inferenceImage, classes, + probThresh, + nmsThresh, + numKps); + } // --- Rescale coordinates from full-res back to display-res --- if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) { @@ -2075,42 +2260,38 @@ namespace ANSCENTER { for (auto& obj : results) obj.cameraId = camera_id; - // Tracking/stabilization (ApplyTracking has its own lock) - if (trackerEnabled && !m_ortEngine->lastWasClassification) { + // Tracking/stabilization (ApplyTracking has its own lock). + // B2: read lastWasClassification from the LOCAL engine copy + // — using m_ortEngine here would re-introduce the race we + // just closed for detect(). + if (trackerEnabled && !engine->lastWasClassification) { results = ApplyTracking(results, camera_id); if (stabilizationEnabled) results = StabilizeDetections(results, camera_id); } + + // A successful inference clears the DML failure budget so + // isolated transient hiccups (one E_FAIL every few minutes) + // never accumulate toward kMaxConsecutiveDmlFailures and + // wrongly demote us to CPU. Only sustained back-to-back + // failures with no successful inference between them + // exhaust the budget. + _dmlConsecutiveFailures.store(0, std::memory_order_relaxed); return results; } catch (const std::exception& e) { const std::string msg = e.what(); - if (msg.find("887A0005") != std::string::npos) { - if (!_dmlDeviceLost) { - _dmlDeviceLost = true; - _logger.LogFatal("ANSONNXYOLO::DetectObjects", - "DirectML GPU device lost (887A0005) — attempting CPU fallback", - __FILE__, __LINE__); - ANS_DBG("ONNXYOLO", "DML device lost — recreating session on CPU"); - try { - std::lock_guard lk(_mutex); - m_ortEngine.reset(); - if (InitOrtEngine(ANSCENTER::EngineType::CPU)) { - _logger.LogInfo("ANSONNXYOLO::DetectObjects", - "CPU fallback session created successfully", - __FILE__, __LINE__); - ANS_DBG("ONNXYOLO", "CPU fallback OK"); - } else { - _logger.LogFatal("ANSONNXYOLO::DetectObjects", - "CPU fallback session creation failed", - __FILE__, __LINE__); - } - } catch (const std::exception& re) { - _logger.LogFatal("ANSONNXYOLO::DetectObjects", - std::string("CPU fallback exception: ") + re.what(), - __FILE__, __LINE__); - } - } + // Any DML-fatal HRESULT or DML branded error → rebuild on CPU. + // See IsDmlFatalError() in ANSEngineCommon.h for the full set + // (887A0005 device-removed, 80004005 generic E_FAIL from + // DmlGraphFusionHelper / DmlCommandRecorder, etc.). + if (IsDmlFatalError(msg)) { + // Stay-on-DML retry — see HandleDmlRunFailure for full + // contract. Tries to rebuild the session on the same + // DML EP first; only falls back to CPU on + // device-removed (887A0005) or after the retry budget + // is exhausted. + HandleDmlRunFailure("ANSONNXYOLO::DetectObjects", msg); return {}; } @@ -2157,19 +2338,9 @@ namespace ANSCENTER { } catch (const std::exception& e) { const std::string msg = e.what(); - if (msg.find("887A0005") != std::string::npos) { - if (!_dmlDeviceLost) { - _dmlDeviceLost = true; - _logger.LogFatal("ANSONNXYOLO::RunInferencesBatch", - "DirectML GPU device lost (887A0005) — attempting CPU fallback", - __FILE__, __LINE__); - try { - m_ortEngine.reset(); - if (!InitOrtEngine(ANSCENTER::EngineType::CPU)) - _logger.LogFatal("ANSONNXYOLO::RunInferencesBatch", - "CPU fallback session creation failed", __FILE__, __LINE__); - } catch (...) {} - } + if (IsDmlFatalError(msg)) { + // Stay-on-DML retry — see HandleDmlRunFailure. + HandleDmlRunFailure("ANSONNXYOLO::RunInferencesBatch", msg); return {}; } _logger.LogFatal("ANSONNXYOLO::RunInferencesBatch", @@ -2184,11 +2355,15 @@ namespace ANSCENTER { try { if (_modelLoading.load()) return {}; - // Snapshot config under brief lock + // Snapshot config + engine handle under brief lock. + // B2: shared_ptr copy `engine` keeps the underlying ONNXYOLO + // alive across the lock-free detectBatch() call even if a + // concurrent DML-fatal handler resets m_ortEngine. float probThresh, nmsThresh; int numKps; std::vector classes; bool trackerEnabled, stabilizationEnabled; + std::shared_ptr engine; { auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjectsBatch"); if (!lk.owns_lock()) return {}; @@ -2204,13 +2379,20 @@ namespace ANSCENTER { classes = _classes; trackerEnabled = _trackerEnabled; stabilizationEnabled = _stabilizationEnabled; + engine = m_ortEngine; // ref-count++ } - // Heavy work outside lock - auto batchResults = m_ortEngine->detectBatch( - inputImages, classes, probThresh, nmsThresh, numKps); + // Heavy work outside lock — dereference local copy, not member. + // Option A: same DmlInferenceGuard contract as DetectObjects. + std::vector> batchResults; + { + DmlInferenceGuard dmlGuard(engine->getEngineType(), + "ANSONNXYOLO::DetectObjectsBatch"); + batchResults = engine->detectBatch( + inputImages, classes, probThresh, nmsThresh, numKps); + } - const bool isClassification = m_ortEngine->lastBatchWasClassification; + const bool isClassification = engine->lastBatchWasClassification; for (auto& results : batchResults) { for (auto& obj : results) @@ -2222,24 +2404,16 @@ namespace ANSCENTER { } } + // Clear DML failure budget on success — see DetectObjects + // for the full rationale. + _dmlConsecutiveFailures.store(0, std::memory_order_relaxed); return batchResults; } catch (const std::exception& e) { const std::string msg = e.what(); - if (msg.find("887A0005") != std::string::npos) { - if (!_dmlDeviceLost) { - _dmlDeviceLost = true; - _logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch", - "DirectML GPU device lost (887A0005) — attempting CPU fallback", - __FILE__, __LINE__); - try { - std::lock_guard lk(_mutex); - m_ortEngine.reset(); - if (!InitOrtEngine(ANSCENTER::EngineType::CPU)) - _logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch", - "CPU fallback session creation failed", __FILE__, __LINE__); - } catch (...) {} - } + if (IsDmlFatalError(msg)) { + // Stay-on-DML retry — see HandleDmlRunFailure. + HandleDmlRunFailure("ANSONNXYOLO::DetectObjectsBatch", msg); return {}; } _logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch", diff --git a/modules/ANSODEngine/ANSONNXYOLO.h b/modules/ANSODEngine/ANSONNXYOLO.h index 1e85136..52794ae 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.h +++ b/modules/ANSODEngine/ANSONNXYOLO.h @@ -210,14 +210,74 @@ namespace ANSCENTER { int NUM_KPS{ 0 }; float KPS_THRESHOLD{ 0.5f }; - // ONNX Runtime inference engine - std::unique_ptr m_ortEngine; + // ONNX Runtime inference engine. + // + // B2: shared_ptr (was unique_ptr) so concurrent inference threads + // can grab a ref-counted local copy under the brief snapshot lock + // and dereference it OUTSIDE the lock without racing with a + // concurrent reset() in the DML-fatal CPU-fallback path. Without + // this, a thread doing `m_ortEngine->detect(...)` lock-free could + // see its underlying object freed mid-call by another thread that + // caught a DML E_FAIL and rebuilt the session — that was the + // 0xc0000005 access violation observed in ANSODEngine.dll under + // AMD/DirectML when multiple AI tasks ran in parallel. + std::shared_ptr m_ortEngine; - // DML device-lost recovery: when DirectML's GPU device is removed - // (HRESULT 887A0005), the session is permanently broken. We detect - // this once, attempt a CPU-fallback recreation, and suppress further - // error-log flooding. - bool _dmlDeviceLost{ false }; + // ──────────────────────────────────────────────────────────────── + // DML stay-on-GPU recovery state. + // + // AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from + // DmlGraphFusionHelper / DmlCommandRecorder when the driver + // chokes under concurrent inference pressure. Empirically this + // is NOT a permanent device loss — recreating the ORT session on + // the same DML EP clears the wedged state and resumes GPU + // inference at full speed. Only the truly-fatal 887A0005 + // (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback. + // + // Recovery contract (used by every Run() catch block): + // 1. compare_exchange on _dmlRecreateInProgress elects one + // thread as the recreate owner. Other threads return {} + // and retry on the next call. + // 2. The owner checks the cooldown — if we recreated in the + // last kDmlRecreateCooldownMs ms, skip (driver is in a + // prolonged bad state, don't thrash). + // 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit + // kMaxConsecutiveDmlFailures recreates without a successful + // inference in between, fall back to CPU as last resort. + // 4. Otherwise reset() + InitOrtEngine() + + // WarmUpEngine() to rebuild on the same GPU EP. + // 5. On successful inference, _dmlConsecutiveFailures resets + // to 0, so isolated transient hiccups don't accumulate + // toward the CPU-fallback threshold. + // + // This replaces the old single-shot `bool _dmlDeviceLost` flag + // which latched on the first failure and forced CPU forever — + // user requirement is to keep AI tasks running on GPU, not to + // trade GPU performance for stability the moment the driver + // hiccups once. + // ──────────────────────────────────────────────────────────────── + std::atomic _dmlRecreateInProgress{ false }; + std::atomic _dmlConsecutiveFailures{ 0 }; + std::chrono::steady_clock::time_point _lastDmlRecreateAt{}; + + // Tunables — tuned for AMD Radeon 780M observed behavior. + // threshold: catches isolated transient AMD DML hiccups. After + // 3 consecutive recreates without a successful inference, the + // driver is in a prolonged bad state and CPU is the only + // forward path. + // cooldown: prevents recreate thrash. AMD DML session creation + // takes 200–500 ms; if E_FAIL fires from many threads at once + // we don't want to recreate N times in a row. + static constexpr int kMaxConsecutiveDmlFailures = 3; + static constexpr int64_t kDmlRecreateCooldownMs = 5000; + + // Centralised DML-fatal recovery used by all 3 catch sites. + // Caller must have already determined the exception is DML-related + // (IsDmlFatalError == true). msg is the exception message used + // both to log and to detect the device-removed sub-case. Returns + // immediately without doing anything when another thread already + // owns the recreate (CAS loser path). + void HandleDmlRunFailure(const char* context, const std::string& msg); // Internal detection pipeline std::vector DetectObjects(const cv::Mat& inputImage, diff --git a/modules/ANSODEngine/ANSOVFaceDetector.cpp b/modules/ANSODEngine/ANSOVFaceDetector.cpp index d43190c..a6faac0 100644 --- a/modules/ANSODEngine/ANSOVFaceDetector.cpp +++ b/modules/ANSODEngine/ANSOVFaceDetector.cpp @@ -384,13 +384,17 @@ namespace ANSCENTER { catch (const std::exception& e) { const std::string msg = e.what(); - // DML device-removal detection (see ANSONNXYOLO.cpp for details) - if (msg.find("887A0005") != std::string::npos) { + // DML fatal-error detection (887A0005 device-removed, 80004005 generic + // DML E_FAIL, branded DmlFusedNode / DmlCommandRecorder / etc.). + // See ANSCENTER::IsDmlFatalError in ANSEngineCommon.h. + if (IsDmlFatalError(msg)) { if (!_dmlDeviceLost) { _dmlDeviceLost = true; _logger.LogFatal("ANSOVFD::RunInference", - "DirectML GPU device lost (887A0005) — will attempt CPU fallback on next call", + std::string("DirectML fatal error — will attempt CPU fallback on next call. msg=") + + msg.substr(0, 256), __FILE__, __LINE__); + ANS_DBG("ANSOVFD", "DML fatal msg=%s", msg.c_str()); } return {}; } diff --git a/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp b/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp index 47e7454..0ab37da 100644 --- a/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp +++ b/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp @@ -16,6 +16,7 @@ #include #include "ANSLPR.h" #include "ANSLPR_CPU.h" +#include "ANSODEngine.h" #include "ANSOpenCV.h" #include "ANSRTSP.h" #include "ANSVideoPlayer.h" @@ -4087,6 +4088,1093 @@ int ALPR_OCR_VideoTest() { return 0; } +// ────────────────────────────────────────────────────────────────────── +// ALPR_Helmet_Parallel_VideoTest — ANSVIS-style multi-task simulation. +// +// Combines the FilePlayer + multi-task fan-out from +// CustomModel_StressTest_FilePlayer (ANSODEngine-UnitTest) with the +// ANSALPR_OCR pipeline from ALPR_OCR_VideoTest, plus an OD-then-batched- +// ALPR pipeline that mirrors how ANSVIS chains a custom detector into +// ALPR for vehicle / scene-aware plate reading. +// +// Task A (ALPR) : ANSALPR_OCR engine (engineType = 2, country = VN) +// on FULL frames via ANSALPR_RunInferenceBinary — +// independent worker, runs in parallel with B. +// Task B (Helmet : custom detector (modelType = 16) via +// + batch ALPR) RunInferenceComplete_CPP. After OD completes, +// any detection whose class name looks like a +// license plate is collected into a cv::Rect list +// and submitted to alprHandle->RunInferencesBatch +// (the same engine method ANSALPR_RunInferencesBatch_LV +// wraps for LabVIEW — see HelmetParallelWorker for +// why we don't go through the _LV entry directly). +// +// alprHandle is shared between A and B; ANSALPR's internal refcount +// (ALPRHandleGuard / AcquireALPRHandle in ANSLPR/dllmain.cpp) makes +// concurrent calls safe. +// +// Per-frame flow per task (mirrors LabVIEW camera→AI plumbing): +// GetFilePlayerCVImage → ANSCV_CloneImage_S → run inference on clone +// → (helmet only) batch ALPR on LP boxes +// → draw on clone → ANSCV_ReleaseImage_S +// +// Press ESC to stop. Edit the CONFIG block below to point at your +// video / model paths. +// ────────────────────────────────────────────────────────────────────── +namespace { + +struct ParallelTaskState { + std::mutex mtx; + cv::Mat displayFrame; + double fps = 0.0; + double inferenceMs = 0.0; + double grabMs = 0.0; + int frameCount = 0; + int detectionCount = 0; + std::string statusMsg = "Initializing"; + std::string lastDetection; + bool engineLoaded = false; +}; + +static std::atomic g_parallelRunning{ true }; + +// ============================================================================ +// AMD/DML root-cause diagnostics — instrumentation shared by both workers. +// Goal: capture enough state per GPU call to differentiate Intel/NVIDIA-OK +// vs AMD-fails behaviour without touching engine code. +// +// Signals captured: +// - Concurrent inference depth (atomic counter incremented at GPU-call +// entry, decremented at exit). On NVIDIA you typically see depth=2 +// routinely (true concurrent kernels). On AMD/DirectML the D3D12 +// driver tends to serialise, so depth>1 here means queue contention — +// compare with NVIDIA logs to see the difference. +// - Per-call rolling latency stats (n / min / mean / p99 / max) for each +// GPU-bound entry point. Progressive degradation precedes the AMD +// 0xE0434352 device-loss; this exposes it. +// - Stuck-call watchdog (>2 s) — Windows TDR fires at 2 s, so anything +// flagged here is a TDR precursor. +// - SLOW(>1s) tag on individual calls. +// - Thread ID + tag on every entry/exit so we can correlate ALPR vs +// Helmet workers in the log timeline. +// - Periodic 5-second stats snapshot to ANS_DBG + stdout. +// ============================================================================ + +static std::atomic g_dmlInflight{ 0 }; // currently-executing GPU calls +static std::atomic g_dmlPeakDepth{ 0 }; // high-water mark of dmlInflight +static std::atomic g_alprFullFrameCalls{ 0 }; +static std::atomic g_helmetOdCalls{ 0 }; +static std::atomic g_batchAlprCalls{ 0 }; +static std::atomic g_slowCalls{ 0 }; // count of calls >1s +static std::atomic g_stuckEvents{ 0 }; // count of watchdog STUCK logs + +// Each in-flight GPU call registers itself here; the watchdog scans the list +// every 500 ms and screams about any entry that's been alive >2 s. +struct InflightCall { + const char* tag; + DWORD tid; + std::chrono::steady_clock::time_point t0; +}; +static std::mutex g_inflightMtx; +static std::vector g_inflight; + +// Bounded rolling latency histogram per call type. +struct LatencyStats { + static constexpr size_t kCap = 500; + std::mutex mtx; + std::deque ms; + void Add(double v) { + std::lock_guard lk(mtx); + ms.push_back(v); + if (ms.size() > kCap) ms.pop_front(); + } + void Snapshot(size_t& n, double& mn, double& mx, + double& mean, double& p99) { + std::lock_guard lk(mtx); + n = ms.size(); + if (n == 0) { mn = mx = mean = p99 = 0.0; return; } + mn = *std::min_element(ms.begin(), ms.end()); + mx = *std::max_element(ms.begin(), ms.end()); + double s = 0.0; for (double v : ms) s += v; + mean = s / static_cast(n); + std::vector sorted(ms.begin(), ms.end()); + std::sort(sorted.begin(), sorted.end()); + size_t idx = std::min(n - 1, static_cast(static_cast(n) * 0.99)); + p99 = sorted[idx]; + } +}; +static LatencyStats g_alprFullLatency; +static LatencyStats g_helmetOdLatency; +static LatencyStats g_batchAlprLatency; + +// RAII scope around a single GPU-bound call. Logs entry+exit, registers +// in the watchdog list, and feeds the rolling latency histogram on exit. +class GpuCallScope { +public: + GpuCallScope(const char* tag, LatencyStats* stats) + : tag_(tag), stats_(stats), + t0_(std::chrono::steady_clock::now()), + tid_(GetCurrentThreadId()) + { + const int depth = ++g_dmlInflight; + // Update peak depth observed (CAS loop — lock-free). + int prev = g_dmlPeakDepth.load(std::memory_order_relaxed); + while (depth > prev && + !g_dmlPeakDepth.compare_exchange_weak(prev, depth)) {} + ANS_DBG("ALPRParallel", "GPU ENTER %s tid=%lu depth=%d", + tag_, static_cast(tid_), depth); + std::lock_guard lk(g_inflightMtx); + g_inflight.push_back({ tag_, tid_, t0_ }); + } + ~GpuCallScope() { + const auto t1 = std::chrono::steady_clock::now(); + const double ms = + std::chrono::duration(t1 - t0_).count(); + const int newDepth = --g_dmlInflight; + if (stats_) stats_->Add(ms); + if (ms > 1000.0) { + g_slowCalls.fetch_add(1, std::memory_order_relaxed); + ANS_DBG("ALPRParallel", + "GPU SLOW(>1s) %s tid=%lu ms=%.1f depth-after=%d slowTotal=%lld", + tag_, static_cast(tid_), ms, newDepth, + g_slowCalls.load()); + } else { + ANS_DBG("ALPRParallel", "GPU exit %s tid=%lu ms=%.1f depth-after=%d", + tag_, static_cast(tid_), ms, newDepth); + } + std::lock_guard lk(g_inflightMtx); + for (auto it = g_inflight.begin(); it != g_inflight.end(); ++it) { + if (it->tid == tid_ && it->t0 == t0_) { + g_inflight.erase(it); + break; + } + } + } + GpuCallScope(const GpuCallScope&) = delete; + GpuCallScope& operator=(const GpuCallScope&) = delete; +private: + const char* tag_; + LatencyStats* stats_; + std::chrono::steady_clock::time_point t0_; + DWORD tid_; +}; + +// Watchdog thread: scans g_inflight every 500 ms and emits a STUCK log +// for any call alive >2 s. Two seconds is the default Windows TDR +// threshold (HKLM\...\GraphicsDrivers\TdrDelay), so anything caught +// here is by definition the precursor to a 0xE0434352 device-loss. +static void GpuWatchdogThread() { + ANS_DBG("ALPRParallel", "Watchdog thread started (TDR threshold=2000ms)"); + while (g_parallelRunning.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lk(g_inflightMtx); + for (const auto& c : g_inflight) { + const double ms = + std::chrono::duration(now - c.t0).count(); + if (ms > 2000.0) { + g_stuckEvents.fetch_add(1, std::memory_order_relaxed); + ANS_DBG("ALPRParallel", + "WATCHDOG STUCK %s tid=%lu inflight=%.0fms (>=2000ms TDR)", + c.tag, static_cast(c.tid), ms); + } + } + } + ANS_DBG("ALPRParallel", "Watchdog thread finished"); +} + +// Periodic stats dumper. Owned by the main test loop; the display thread +// calls Maybe() every render frame and gets a snapshot logged ~every 5 s. +struct StatsSnapshotter { + void Maybe(std::chrono::steady_clock::time_point now) { + if (now - last_ < std::chrono::seconds(5)) return; + last_ = now; + size_t n; double mn, mx, mean, p99; + char buf[1024]; + char* p = buf; + char* const end = buf + sizeof(buf); + p += snprintf(p, end - p, + "STATS dmlInflight=%d peakDepth=%d " + "alprFullCalls=%lld helmetOdCalls=%lld batchAlprCalls=%lld " + "slowCalls=%lld stuckEvents=%lld | ", + g_dmlInflight.load(), g_dmlPeakDepth.load(), + g_alprFullFrameCalls.load(), + g_helmetOdCalls.load(), + g_batchAlprCalls.load(), + g_slowCalls.load(), + g_stuckEvents.load()); + g_alprFullLatency.Snapshot(n, mn, mx, mean, p99); + p += snprintf(p, end - p, + "alprFull(n=%zu mn=%.0f mean=%.0f p99=%.0f mx=%.0f) ", + n, mn, mean, p99, mx); + g_helmetOdLatency.Snapshot(n, mn, mx, mean, p99); + p += snprintf(p, end - p, + "helmetOD(n=%zu mn=%.0f mean=%.0f p99=%.0f mx=%.0f) ", + n, mn, mean, p99, mx); + g_batchAlprLatency.Snapshot(n, mn, mx, mean, p99); + p += snprintf(p, end - p, + "batchALPR(n=%zu mn=%.0f mean=%.0f p99=%.0f mx=%.0f)", + n, mn, mean, p99, mx); + ANS_DBG("ALPRParallel", "%s", buf); + printf("[ALPRParallel] %s\n", buf); + } +private: + std::chrono::steady_clock::time_point last_ = + std::chrono::steady_clock::now(); +}; +static StatsSnapshotter g_statsSnap; + +// ALPR worker: pulls frames from FilePlayer, runs ANSALPR_RunInferenceBinary, +// parses the JSON, draws boxes + plate text, updates display state. +static void ALPRParallelWorker(ANSCENTER::ANSFILEPLAYER* fpClient, + ANSCENTER::ANSALPR* alprHandle, + ParallelTaskState& state) { + const char* tag = "[ALPR ]"; + printf("%s Worker thread started\n", tag); + ANS_DBG("ALPRParallel", "ALPR worker started: fpClient=%p alprHandle=%p", + (void*)fpClient, (void*)alprHandle); + + int width = 0, height = 0; + int64_t pts = 0; + int emptyFrames = 0; + boost::property_tree::ptree pt; + std::deque fpsTimestamps; + + while (g_parallelRunning.load()) { + auto grabStart = std::chrono::steady_clock::now(); + cv::Mat* framePtr = nullptr; + GetFilePlayerCVImage(&fpClient, width, height, pts, &framePtr); + auto grabEnd = std::chrono::steady_clock::now(); + double grabMs = std::chrono::duration(grabEnd - grabStart).count(); + + if (!framePtr || framePtr->empty()) { + emptyFrames++; + if (emptyFrames > 500) { + printf("%s Too many empty frames (%d), stopping\n", tag, emptyFrames); + ANS_DBG("ALPRParallel", "ALPR worker stopping: too many empty frames=%d", emptyFrames); + break; + } + if (framePtr) ANSCV_ReleaseImage_S(&framePtr); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + emptyFrames = 0; + + cv::Mat* clonedImage = nullptr; + int cloneResult = ANSCV_CloneImage_S(&framePtr, &clonedImage); + if (cloneResult != 1 || !clonedImage) { + printf("%s CloneImage failed (result=%d)\n", tag, cloneResult); + ANS_DBG("ALPRParallel", "ANSCV_CloneImage_S failed: result=%d", cloneResult); + ANSCV_ReleaseImage_S(&framePtr); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + ANSCV_ReleaseImage_S(&framePtr); + + // Pack BGR bytes for ANSALPR_RunInferenceBinary (same as ALPR_OCR_VideoTest). + const int frameW = clonedImage->cols; + const int frameH = clonedImage->rows; + const unsigned int bufferLength = + static_cast(clonedImage->total() * clonedImage->elemSize()); + std::vector imageBytes(bufferLength); + std::memcpy(imageBytes.data(), clonedImage->data, bufferLength); + + auto infStart = std::chrono::steady_clock::now(); + std::string detectionResult; + { + GpuCallScope gpu("alprFullFrame", &g_alprFullLatency); + detectionResult = ANSALPR_RunInferenceBinary(&alprHandle, + imageBytes.data(), frameW, frameH); + } + g_alprFullFrameCalls.fetch_add(1, std::memory_order_relaxed); + auto infEnd = std::chrono::steady_clock::now(); + double infMs = std::chrono::duration(infEnd - infStart).count(); + + // Draw on the clone for display. + cv::Mat display = clonedImage->clone(); + std::string lastDet; + int detCount = 0; + if (!detectionResult.empty()) { + try { + pt.clear(); + std::stringstream ss(detectionResult); + boost::property_tree::read_json(ss, pt); + auto resultsChild = pt.get_child_optional("results"); + if (resultsChild) { + for (const auto& child : *resultsChild) { + const auto& res = child.second; + const auto class_name_raw = GetData(res, "class_name"); + const std::string class_name = DecodeUnicodeEscapes(class_name_raw); + const auto x = GetData(res, "x"); + const auto y = GetData(res, "y"); + const auto w = GetData(res, "width"); + const auto h = GetData(res, "height"); + cv::rectangle(display, cv::Rect(x, y, w, h), + cv::Scalar(0, 255, 0), 2); +#ifdef WIN32 + { + int textH = (int)(1.2 * 30); + int ty = y - 5 - textH; + if (ty < 0) ty = y + 3; + putTextUnicode(display, class_name, cv::Point(x, ty), + 1.2, cv::Scalar(0, 0, 255), 2); + } +#else + cv::putText(display, class_name, cv::Point(x, y - 5), + cv::FONT_HERSHEY_SIMPLEX, 0.8, + cv::Scalar(0, 0, 255), 2, cv::LINE_AA); +#endif + if (!class_name.empty()) lastDet = class_name; + ++detCount; + } + } + } catch (const std::exception& e) { + std::cerr << tag << " JSON parse error: " << e.what() << std::endl; + ANS_DBG("ALPRParallel", "JSON parse error: %s", e.what()); + } + } + + ANSCV_ReleaseImage_S(&clonedImage); + + auto now = std::chrono::steady_clock::now(); + fpsTimestamps.push_back(now); + while (!fpsTimestamps.empty() && + std::chrono::duration(now - fpsTimestamps.front()).count() > 2.0) { + fpsTimestamps.pop_front(); + } + double fps = fpsTimestamps.size() / 2.0; + + { + std::lock_guard lk(state.mtx); + if (!display.empty()) state.displayFrame = display; + state.fps = fps; + state.inferenceMs = infMs; + state.grabMs = grabMs; + state.frameCount += 1; + state.detectionCount += detCount; + if (!lastDet.empty()) state.lastDetection = lastDet; + } + + if ((state.frameCount % 200) == 0) { + printf("%s %d frames | %.1f FPS | Inf: %.0f ms | Det: %d\n", + tag, state.frameCount, fps, infMs, state.detectionCount); + } + } + + printf("%s Worker thread finished (%d frames)\n", tag, state.frameCount); + ANS_DBG("ALPRParallel", "ALPR worker finished: frames=%d detections=%d", + state.frameCount, state.detectionCount); +} + +// Case-insensitive substring match on a class name — keeps the LP filter +// tolerant to label variations across model versions ("license_plate", +// "License Plate", "LP", "plate", etc.). +static bool ClassNameLooksLikeLicensePlate(const std::string& className) { + std::string lower; + lower.reserve(className.size()); + for (char c : className) lower.push_back((char)std::tolower((unsigned char)c)); + if (lower.find("plate") != std::string::npos) return true; + if (lower.find("license") != std::string::npos) return true; + if (lower == "lp" || lower == "lpn" || lower == "anpr") return true; + return false; +} + +// Helmet worker: pulls frames from the SAME FilePlayer, runs the custom +// detector via RunInferenceComplete_CPP (same code path LabVIEW takes), then +// — for any detections that look like license plates — runs a single batched +// ALPR call on those crops via ANSALPR::RunInferencesBatch (the C++ method +// that ANSALPR_RunInferencesBatch_LV wraps in production). This is the +// "OD-then-batched-ALPR" pipeline ANSVIS uses on top of the standalone +// full-frame ALPR worker, so both paths exercise alprHandle concurrently. +// +// Note on `_LV` vs C++: +// ANSALPR_RunInferencesBatch_LV serialises results into an LStrHandle +// (LabVIEW string), which requires DSSetHandleSize from the LabVIEW +// runtime — not available in a console test. The _LV wrapper is only +// ~30 lines of LStrHandle plumbing around `engine->RunInferencesBatch`, +// so calling that method directly here exercises the identical inference +// code path without dragging in lvrt.dll. +static void HelmetParallelWorker(ANSCENTER::ANSFILEPLAYER* fpClient, + ANSCENTER::ANSODBase* odHandle, + ANSCENTER::ANSALPR* alprHandle, + ParallelTaskState& state) { + const char* tag = "[Helmet]"; + printf("%s Worker thread started\n", tag); + ANS_DBG("ALPRParallel", "Helmet worker started: fpClient=%p odHandle=%p alprHandle=%p", + (void*)fpClient, (void*)odHandle, (void*)alprHandle); + + int width = 0, height = 0; + int64_t pts = 0; + int emptyFrames = 0; + const std::string cameraId = "ParallelHelmetCam"; + std::deque fpsTimestamps; + long long batchAlprCalls = 0; + long long batchAlprPlatesFound = 0; + + while (g_parallelRunning.load()) { + auto grabStart = std::chrono::steady_clock::now(); + cv::Mat* framePtr = nullptr; + GetFilePlayerCVImage(&fpClient, width, height, pts, &framePtr); + auto grabEnd = std::chrono::steady_clock::now(); + double grabMs = std::chrono::duration(grabEnd - grabStart).count(); + + if (!framePtr || framePtr->empty()) { + emptyFrames++; + if (emptyFrames > 500) { + printf("%s Too many empty frames (%d), stopping\n", tag, emptyFrames); + ANS_DBG("ALPRParallel", "Helmet worker stopping: too many empty frames=%d", emptyFrames); + break; + } + if (framePtr) ANSCV_ReleaseImage_S(&framePtr); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + emptyFrames = 0; + + cv::Mat* clonedImage = nullptr; + int cloneResult = ANSCV_CloneImage_S(&framePtr, &clonedImage); + if (cloneResult != 1 || !clonedImage) { + printf("%s CloneImage failed (result=%d)\n", tag, cloneResult); + ANS_DBG("ALPRParallel", "ANSCV_CloneImage_S failed: result=%d", cloneResult); + ANSCV_ReleaseImage_S(&framePtr); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + ANSCV_ReleaseImage_S(&framePtr); + + // --- Stage 1: helmet OD --- + auto infStart = std::chrono::steady_clock::now(); + std::vector detections; + int infResult = 0; + { + GpuCallScope gpu("helmetOD", &g_helmetOdLatency); + infResult = RunInferenceComplete_CPP(&odHandle, &clonedImage, + cameraId.c_str(), "", detections); + } + g_helmetOdCalls.fetch_add(1, std::memory_order_relaxed); + auto infEnd = std::chrono::steady_clock::now(); + double infMs = std::chrono::duration(infEnd - infStart).count(); + + // --- Stage 2: filter LP-like detections, run batched ALPR on them --- + std::vector lpBoxes; + if (infResult > 0) { + lpBoxes.reserve(detections.size()); + for (const auto& obj : detections) { + if (ClassNameLooksLikeLicensePlate(obj.className)) { + if (obj.box.width > 0 && obj.box.height > 0) { + lpBoxes.push_back(obj.box); + } + } + } + } + + std::vector plateResults; + double batchAlprMs = 0.0; + if (!lpBoxes.empty() && alprHandle && clonedImage && !clonedImage->empty()) { + ANS_DBG("ALPRParallel", "Batch ALPR call: lpBoxes=%zu", lpBoxes.size()); + auto t0 = std::chrono::steady_clock::now(); + try { + GpuCallScope gpu("batchALPR", &g_batchAlprLatency); + plateResults = alprHandle->RunInferencesBatch( + *clonedImage, lpBoxes, cameraId); + } catch (const std::exception& e) { + ANS_DBG("ALPRParallel", "RunInferencesBatch threw: %s", e.what()); + } catch (...) { + ANS_DBG("ALPRParallel", "RunInferencesBatch threw unknown exception"); + } + g_batchAlprCalls.fetch_add(1, std::memory_order_relaxed); + auto t1 = std::chrono::steady_clock::now(); + batchAlprMs = std::chrono::duration(t1 - t0).count(); + batchAlprCalls += 1; + batchAlprPlatesFound += static_cast(plateResults.size()); + ANS_DBG("ALPRParallel", + "Batch ALPR DONE: %zu LP boxes -> %zu plates, %.1f ms", + lpBoxes.size(), plateResults.size(), batchAlprMs); + } + + // --- Draw helmet boxes (yellow) + plate boxes (green w/ OCR text) --- + // Important: when the helmet OD detects a license-plate-class object, + // its `className` is the literal string "license plate" — NOT a + // recognized plate number. We deliberately SKIP the OD label for + // LP-class detections; the batch-ALPR loop below redraws those boxes + // with the OCR'd plate string when ANSALPR succeeds. If ANSALPR + // fails to read a plate, the green box still flags where the helmet + // OD found one, but no misleading "license plate" text is drawn. + cv::Mat display; + if (clonedImage && !clonedImage->empty()) display = clonedImage->clone(); + + std::string lastDet; // last NON-LP helmet class (helmet, no_helmet, motorcyclist...) + std::string lastPlateText; // last successfully OCR'd plate string + int detCount = 0; + if (infResult > 0) { + for (const auto& obj : detections) { + ++detCount; + if (ClassNameLooksLikeLicensePlate(obj.className)) { + // Defer to the batch-ALPR loop — it owns LP visualisation. + continue; + } + if (!display.empty()) { + cv::rectangle(display, obj.box, cv::Scalar(0, 255, 255), 2); + std::string label = obj.className + " " + + std::to_string((int)(obj.confidence * 100)) + "%"; + cv::putText(display, label, + cv::Point(obj.box.x, obj.box.y - 5), + cv::FONT_HERSHEY_SIMPLEX, 0.6, + cv::Scalar(0, 255, 255), 2); + } + lastDet = obj.className; + } + } + for (const auto& p : plateResults) { + if (display.empty()) break; + cv::rectangle(display, p.box, cv::Scalar(0, 255, 0), 2); + const std::string plateText = DecodeUnicodeEscapes(p.className); + if (!plateText.empty()) { +#ifdef WIN32 + int ty = p.box.y - 5 - (int)(1.0 * 30); + if (ty < 0) ty = p.box.y + 3; + putTextUnicode(display, plateText, cv::Point(p.box.x, ty), + 1.0, cv::Scalar(0, 0, 255), 2); +#else + cv::putText(display, plateText, + cv::Point(p.box.x, std::max(0, p.box.y - 5)), + cv::FONT_HERSHEY_SIMPLEX, 0.8, + cv::Scalar(0, 0, 255), 2, cv::LINE_AA); +#endif + lastPlateText = plateText; + } + } + // Status-bar text: prefer the OCR'd plate over a non-LP helmet class. + // Both are kept so the cell stays informative when the OD detected + // (e.g.) a helmet but no plate text was recognised this frame. + std::string statusDet; + if (!lastPlateText.empty()) statusDet = "PLATE: " + lastPlateText; + else if (!lastDet.empty()) statusDet = lastDet; + // Overlay "ALPR-from-OD: calls, plates" badge + if (!display.empty() && batchAlprCalls > 0) { + char badge[160]; + snprintf(badge, sizeof(badge), + "ALPR-from-OD: %lld calls, %lld plates (last: %.1f ms)", + batchAlprCalls, batchAlprPlatesFound, batchAlprMs); + cv::putText(display, badge, cv::Point(8, 50), + cv::FONT_HERSHEY_SIMPLEX, 0.55, + cv::Scalar(0, 255, 0), 2); + } + + ANSCV_ReleaseImage_S(&clonedImage); + + auto now = std::chrono::steady_clock::now(); + fpsTimestamps.push_back(now); + while (!fpsTimestamps.empty() && + std::chrono::duration(now - fpsTimestamps.front()).count() > 2.0) { + fpsTimestamps.pop_front(); + } + double fps = fpsTimestamps.size() / 2.0; + + { + std::lock_guard lk(state.mtx); + if (!display.empty()) state.displayFrame = display; + state.fps = fps; + state.inferenceMs = infMs + batchAlprMs; // total cost per OD frame + state.grabMs = grabMs; + state.frameCount += 1; + state.detectionCount += detCount + (int)plateResults.size(); + if (!statusDet.empty()) state.lastDetection = statusDet; + } + + if ((state.frameCount % 200) == 0) { + printf("%s %d frames | %.1f FPS | OD: %.0f ms | Batch ALPR: %.0f ms (%lld calls, %lld plates) | Det: %d | last:%s\n", + tag, state.frameCount, fps, infMs, batchAlprMs, + batchAlprCalls, batchAlprPlatesFound, state.detectionCount, + statusDet.empty() ? "-" : statusDet.c_str()); + } + } + + printf("%s Worker thread finished (%d frames, %lld batch-ALPR calls -> %lld plates)\n", + tag, state.frameCount, batchAlprCalls, batchAlprPlatesFound); + ANS_DBG("ALPRParallel", + "Helmet worker finished: frames=%d detections=%d batchCalls=%lld batchPlates=%lld", + state.frameCount, state.detectionCount, batchAlprCalls, batchAlprPlatesFound); +} + +} // namespace + +int ALPR_Helmet_Parallel_VideoTest() { + printf("\n"); + printf("============================================================\n"); + printf(" ALPR + Helmet Parallel Video Test (ANSVIS simulation)\n"); + printf(" 1 FilePlayer feeds 2 AI tasks running concurrently:\n"); + printf(" A) ANSALPR_OCR full-frame (engineType=2, country=VIETNAM)\n"); + printf(" B) Custom helmet detector (modelType=16)\n"); + printf(" └─ on LP-class detections, batched ALPR via\n"); + printf(" ANSALPR::RunInferencesBatch (= the C++ method\n"); + printf(" ANSALPR_RunInferencesBatch_LV calls internally)\n"); + printf(" Press ESC to stop\n"); + printf("============================================================\n\n"); + ANS_DBG("ALPRParallel", "ALPR_Helmet_Parallel_VideoTest START"); + + // ===== CONFIG (edit to match your environment) ===================== + const std::string videoFilePath = + "C:\\ProgramData\\ANSCENTER\\Shared\\HM1.mp4"; + + // ALPR (ANSALPR_OCR) + const std::string alprModelPath = + "C:\\ProgramData\\ANSCENTER\\ANSVIS Server\\ANSALPR\\ANS_GenericALPR_v2.0.zip"; + const int alprEngineType = 2; // 2 = ANSALPR_OCR + const int alprCountry = 0; // 0 = VIETNAM + const double alprDetectionThresh = 0.3; + const double alprOcrThresh = 0.5; + const double alprColourThresh = 0.0; + + // Helmet (custom detector — same model the OD stress test uses) + const std::string helmetModelPath = + "C:\\ProgramData\\ANSCENTER\\ANSVIS Server\\Models\\B-IN_ANS_Helmet_v2.0_102728911.zip"; + const int helmetModelType = 16; // 16 = CustomDetector + const int helmetDetectorType = 1; // 1 = Detection + const float helmetScoreThresh = 0.5f; + const float helmetConfThresh = 0.5f; + const float helmetNmsThresh = 0.45f; + // =================================================================== + + g_parallelRunning.store(true); + + ANSCENTER::ANSOPENCV::InitCameraNetwork(); + + // --- Optimize helmet model (warmup engine cache, same as OD stress test) --- + std::cout << "Optimizing helmet model, please wait..." << std::endl; + std::string optimizedFolder; + OptimizeModelStr(helmetModelPath.c_str(), "", + helmetModelType, helmetDetectorType, /*fp16=*/1, + optimizedFolder); + std::cout << "Optimized helmet model folder: " << optimizedFolder << std::endl; + ANS_DBG("ALPRParallel", "Helmet model optimize done: folder=%s", + optimizedFolder.c_str()); + + // --- Create FilePlayer (single source feeds both tasks) --- + ANSCENTER::ANSFILEPLAYER* fpClient = nullptr; + { + printf("[Stream] Creating FilePlayer: %s\n", videoFilePath.c_str()); + int result = CreateANSFilePlayerHandle(&fpClient, "", videoFilePath.c_str()); + if (result != 1 || !fpClient) { + printf("[Stream] FAILED to create FilePlayer (result=%d)\n", result); + ANS_DBG("ALPRParallel", "CreateANSFilePlayerHandle FAIL result=%d", result); + ANSCENTER::ANSOPENCV::DeinitCameraNetwork(); + return -1; + } + SetFilePlayerDisplayResolution(&fpClient, 1920, 1080); + printf("[Stream] FilePlayer created (display: 1920x1080)\n"); + } + + // --- Create ALPR handle --- + ANSCENTER::ANSALPR* alprHandle = nullptr; + { + printf("[ALPR ] Creating ANSALPR handle (engineType=%d)...\n", alprEngineType); + int created = CreateANSALPRHandle(&alprHandle, "", + alprModelPath.c_str(), "", + alprEngineType, alprDetectionThresh, alprOcrThresh, alprColourThresh); + if (!created || !alprHandle) { + printf("[ALPR ] FAILED to create ANSALPR handle\n"); + ANS_DBG("ALPRParallel", "CreateANSALPRHandle FAIL"); + StopFilePlayer(&fpClient); + ReleaseANSFilePlayerHandle(&fpClient); + ANSCENTER::ANSOPENCV::DeinitCameraNetwork(); + return -2; + } + ANSALPR_SetCountry(&alprHandle, alprCountry); + auto loadStart = std::chrono::steady_clock::now(); + int loadResult = LoadANSALPREngineHandle(&alprHandle); + auto loadEnd = std::chrono::steady_clock::now(); + double loadMs = std::chrono::duration(loadEnd - loadStart).count(); + if (!loadResult) { + printf("[ALPR ] FAILED to load ANSALPR engine\n"); + ANS_DBG("ALPRParallel", "LoadANSALPREngineHandle FAIL"); + ReleaseANSALPRHandle(&alprHandle); + StopFilePlayer(&fpClient); + ReleaseANSFilePlayerHandle(&fpClient); + ANSCENTER::ANSOPENCV::DeinitCameraNetwork(); + return -3; + } + printf("[ALPR ] Engine loaded in %.0f ms (country=%d)\n", loadMs, alprCountry); + ANS_DBG("ALPRParallel", "ALPR engine loaded: %.0f ms country=%d", loadMs, alprCountry); + } + + // --- Create helmet OD handle (same API as VideoDetectorEngine / LabVIEW) --- + ANSCENTER::ANSODBase* odHandle = nullptr; + { + printf("[Helmet] Creating OD handle (modelType=%d)...\n", helmetModelType); + auto loadStart = std::chrono::steady_clock::now(); + std::string labelMap = CreateANSODHandle( + &odHandle, "", + helmetModelPath.c_str(), "", + helmetScoreThresh, helmetConfThresh, helmetNmsThresh, + /*autoDetectEngine=*/1, helmetModelType, helmetDetectorType, + /*loadEngineOnCreation=*/1); + auto loadEnd = std::chrono::steady_clock::now(); + double loadMs = std::chrono::duration(loadEnd - loadStart).count(); + if (!odHandle) { + printf("[Helmet] FAILED to create OD handle\n"); + ANS_DBG("ALPRParallel", "CreateANSODHandle FAIL"); + ReleaseANSALPRHandle(&alprHandle); + StopFilePlayer(&fpClient); + ReleaseANSFilePlayerHandle(&fpClient); + ANSCENTER::ANSOPENCV::DeinitCameraNetwork(); + return -4; + } + // Print the FULL label map — needed so the operator can confirm the + // helmet model exposes a license-plate class for the OD-then-batched- + // ALPR pipeline (HelmetParallelWorker filters by class name). + printf("[Helmet] Model loaded in %.0f ms\n", loadMs); + printf("[Helmet] Full label map: %s\n", + labelMap.empty() ? "(none)" : labelMap.c_str()); + ANS_DBG("ALPRParallel", "Helmet OD handle created: %.0f ms labelLen=%zu labels=%s", + loadMs, labelMap.size(), + labelMap.empty() ? "(none)" : labelMap.c_str()); + } + + // --- Start playback --- + StartFilePlayer(&fpClient); + printf("[Stream] Playback started\n"); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // --- Launch worker threads (both pull from the same FilePlayer) --- + ParallelTaskState alprState; + ParallelTaskState helmetState; + { + std::lock_guard lkA(alprState.mtx); + alprState.engineLoaded = true; + alprState.statusMsg = "Running"; + std::lock_guard lkH(helmetState.mtx); + helmetState.engineLoaded = true; + helmetState.statusMsg = "Running"; + } + + // Helmet worker shares alprHandle with the standalone ALPR worker — both + // call into ANSALPR concurrently (full-frame in ALPRParallelWorker; + // batched on OD-supplied LP boxes in HelmetParallelWorker). ANSALPR is + // refcounted internally (see ALPRHandleGuard / AcquireALPRHandle in + // ANSLPR/dllmain.cpp) so concurrent use is safe. + std::thread alprWorker(ALPRParallelWorker, fpClient, alprHandle, std::ref(alprState)); + std::thread helmetWorker(HelmetParallelWorker, fpClient, odHandle, alprHandle, + std::ref(helmetState)); + // Watchdog thread: scans in-flight GPU calls every 500 ms and screams + // about anything alive >2 s (the Windows TDR threshold). Captures the + // exact precursor to the AMD 0xE0434352 device-loss. + std::thread watchdog(GpuWatchdogThread); + + // --- Display loop: 2 cells side-by-side --- + const int cellW = 720, cellH = 480; + const char* windowName = "ALPR + Helmet Parallel (ANSVIS sim)"; + cv::namedWindow(windowName, cv::WINDOW_NORMAL); + cv::resizeWindow(windowName, cellW * 2, cellH + 40); + + auto testStart = std::chrono::steady_clock::now(); + + auto renderCell = [cellW, cellH](ParallelTaskState& s, const char* title, + cv::Scalar accent) -> cv::Mat { + cv::Mat cell; + double fps = 0, infMs = 0; + int fCount = 0, dCount = 0; + std::string statusMsg, lastDet; + bool engineLoaded = false; + { + std::lock_guard lk(s.mtx); + if (!s.displayFrame.empty()) { + cv::resize(s.displayFrame, cell, cv::Size(cellW, cellH)); + } + fps = s.fps; + infMs = s.inferenceMs; + fCount = s.frameCount; + dCount = s.detectionCount; + statusMsg = s.statusMsg; + lastDet = s.lastDetection; + engineLoaded = s.engineLoaded; + } + if (cell.empty()) { + cell = cv::Mat(cellH, cellW, CV_8UC3, cv::Scalar(40, 40, 40)); + cv::putText(cell, std::string(title) + ": " + statusMsg, + cv::Point(20, cellH / 2), + cv::FONT_HERSHEY_SIMPLEX, 0.8, + cv::Scalar(100, 100, 255), 2); + } + cv::rectangle(cell, cv::Rect(0, 0, cellW, 25), + cv::Scalar(0, 0, 0), cv::FILLED); + cv::putText(cell, title, cv::Point(8, 18), + cv::FONT_HERSHEY_SIMPLEX, 0.55, accent, 2); + cv::rectangle(cell, cv::Rect(0, cellH - 45, cellW, 45), + cv::Scalar(0, 0, 0), cv::FILLED); + char bar1[256], bar2[256]; + snprintf(bar1, sizeof(bar1), + "%.1f FPS | inf:%.0fms | Frames:%d | Det:%d", + fps, infMs, fCount, dCount); + snprintf(bar2, sizeof(bar2), "last:%s", + lastDet.empty() ? "-" : lastDet.c_str()); + cv::Scalar barColor = engineLoaded ? cv::Scalar(0, 255, 0) + : cv::Scalar(0, 100, 255); + cv::putText(cell, bar1, cv::Point(5, cellH - 25), + cv::FONT_HERSHEY_SIMPLEX, 0.5, barColor, 1); + cv::putText(cell, bar2, cv::Point(5, cellH - 5), + cv::FONT_HERSHEY_SIMPLEX, 0.5, + cv::Scalar(0, 200, 255), 1); + return cell; + }; + + while (g_parallelRunning.load()) { + // Periodic AMD/DML diagnostics dump — flushes latency histograms + // and inflight depth to ANS_DBG + stdout every ~5 s. + g_statsSnap.Maybe(std::chrono::steady_clock::now()); + + cv::Mat canvas(cellH + 40, cellW * 2, CV_8UC3, cv::Scalar(30, 30, 30)); + + cv::Mat alprCell = renderCell(alprState, "ALPR full-frame (ANSALPR_OCR)", + cv::Scalar(0, 200, 255)); + cv::Mat helmetCell = renderCell(helmetState, "Helmet OD + batched ALPR-on-OD", + cv::Scalar(0, 255, 255)); + alprCell .copyTo(canvas(cv::Rect(0, 0, cellW, cellH))); + helmetCell.copyTo(canvas(cv::Rect(cellW, 0, cellW, cellH))); + + double elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - testStart).count(); + double totalFps = alprState.fps + helmetState.fps; + char bottomBar[320]; + snprintf(bottomBar, sizeof(bottomBar), + "Elapsed: %.0fs | Combined: %.1f FPS | depth=%d peak=%d slow=%lld stuck=%lld | ESC to stop", + elapsed, totalFps, + g_dmlInflight.load(), g_dmlPeakDepth.load(), + g_slowCalls.load(), g_stuckEvents.load()); + cv::putText(canvas, bottomBar, cv::Point(10, cellH + 25), + cv::FONT_HERSHEY_SIMPLEX, 0.5, + cv::Scalar(200, 200, 200), 1); + + cv::imshow(windowName, canvas); + int key = cv::waitKey(30); + if (key == 27) { + printf("\nESC pressed — stopping...\n"); + ANS_DBG("ALPRParallel", "ESC pressed, requesting stop"); + g_parallelRunning.store(false); + } + } + + // --- Wait for workers --- + printf("Waiting for worker threads...\n"); + if (alprWorker.joinable()) alprWorker.join(); + if (helmetWorker.joinable()) helmetWorker.join(); + if (watchdog.joinable()) watchdog.join(); + + double totalElapsed = std::chrono::duration( + std::chrono::steady_clock::now() - testStart).count(); + printf("\n============================================================\n"); + printf(" PARALLEL TEST SUMMARY (runtime: %.0fs)\n", totalElapsed); + printf("============================================================\n"); + printf(" ALPR : %d frames | %d detections | %.1f FPS | Inf: %.0fms\n", + alprState.frameCount, alprState.detectionCount, + alprState.fps, alprState.inferenceMs); + printf(" Helmet: %d frames | %d detections | %.1f FPS | Inf: %.0fms\n", + helmetState.frameCount, helmetState.detectionCount, + helmetState.fps, helmetState.inferenceMs); + printf(" Combined throughput: %.1f FPS\n", + alprState.fps + helmetState.fps); + printf("------------------------------------------------------------\n"); + printf(" AMD/DML diagnostics (compare with Intel/NVIDIA run)\n"); + printf(" Peak inflight depth : %d (NVIDIA usually >=2; AMD usually 1)\n", + g_dmlPeakDepth.load()); + printf(" SLOW(>1s) calls total : %lld\n", g_slowCalls.load()); + printf(" Watchdog STUCK events : %lld (each = >2s in flight, TDR precursor)\n", + g_stuckEvents.load()); + { + size_t n; double mn, mx, mean, p99; + g_alprFullLatency.Snapshot(n, mn, mx, mean, p99); + printf(" alprFull latency : n=%zu min=%.0f mean=%.0f p99=%.0f max=%.0f ms\n", + n, mn, mean, p99, mx); + g_helmetOdLatency.Snapshot(n, mn, mx, mean, p99); + printf(" helmetOD latency : n=%zu min=%.0f mean=%.0f p99=%.0f max=%.0f ms\n", + n, mn, mean, p99, mx); + g_batchAlprLatency.Snapshot(n, mn, mx, mean, p99); + printf(" batchALPR latency : n=%zu min=%.0f mean=%.0f p99=%.0f max=%.0f ms\n", + n, mn, mean, p99, mx); + } + printf("============================================================\n"); + ANS_DBG("ALPRParallel", + "TEST END runtime=%.0fs peakDepth=%d slow=%lld stuck=%lld " + "ALPR(frames=%d det=%d) HELMET(frames=%d det=%d)", + totalElapsed, g_dmlPeakDepth.load(), + g_slowCalls.load(), g_stuckEvents.load(), + alprState.frameCount, alprState.detectionCount, + helmetState.frameCount, helmetState.detectionCount); + + // --- Release --- + if (odHandle) ReleaseANSODHandle(&odHandle); + if (alprHandle) ReleaseANSALPRHandle(&alprHandle); + if (fpClient) { + StopFilePlayer(&fpClient); + ReleaseANSFilePlayerHandle(&fpClient); + } + + cv::destroyAllWindows(); + ANSCENTER::ANSOPENCV::DeinitCameraNetwork(); + return 0; +} + +// ────────────────────────────────────────────────────────────────────── +// ALPR_OCR_FolderInferences_Test — folder-driven counterpart of +// ALPR_OCR_VideoTest. Walks the same image folder ANSLPR_OD_FolderInferences_Test +// uses but routes inference through the ANSALPR_OCR engine (engineType=2) +// and the same generic-ALPR model that ALPR_OCR_VideoTest loads. Country +// is set to VIETNAM so the Vietnam-only post-processing +// (alnum filter + uppercase + 11-pattern format validation) is exercised. +// +// Output format mirrors ANSLPR_OD_FolderInferences_Test exactly: one block +// per image with raw JSON, parsed detections, and a tail summary with +// total detections + average inference time. Compare side-by-side with +// the OD result on the same folder to A/B-test OCR vs OD on Vietnam input. +// ────────────────────────────────────────────────────────────────────── +int ALPR_OCR_FolderInferences_Test() { + std::cout << "=== ALPR_OCR_FolderInferences_Test: ANSALPR_OCR engine on folder ===" << std::endl; + std::filesystem::path currentPath = std::filesystem::current_path(); + std::cout << "Current working directory: " << currentPath << std::endl; + + boost::property_tree::ptree pt; + ANSCENTER::ANSALPR* infHandle = nullptr; + std::string licenseKey = ""; + + // Same model the video test uses (ANSALPR_OCR generic Paddle-based model). + std::string modelZipFile = "C:\\ProgramData\\ANSCENTER\\ANSVIS Server\\ANSALPR\\ANS_GenericALPR_v2.0.zip"; + + // Same folder ANSLPR_OD_FolderInferences_Test scans. + std::string folderPath = "E:\\Programs\\DemoAssets\\ImageSeries\\ALPR character M\\EventDebug_20260419_151452.608"; + + int engineType = 2; // 2 = ANSALPR_OCR (Paddle-based OCR pipeline) + double detectionThreshold = 0.3; + double ocrThreshold = 0.5; + double colourThreshold = 0.0; + + // Step 1: Create handle. + int createResult = CreateANSALPRHandle(&infHandle, licenseKey.c_str(), + modelZipFile.c_str(), "", engineType, + detectionThreshold, ocrThreshold, colourThreshold); + std::cout << "CreateANSALPRHandle result: " << createResult << std::endl; + if (!createResult || !infHandle) { + std::cerr << "Failed to create ANSALPR_OCR handle" << std::endl; + return -1; + } + + // Step 2: Country = VIETNAM (0). Activates Vietnam-only post-processing + // (alnum filter + uppercase + format validation against the 11 standard + // Vietnam patterns). Country codes: + // 0 = VIETNAM, 1 = CHINA, 2 = AUSTRALIA, 3 = USA, 4 = INDONESIA, 5 = JAPAN. + ANSALPR_SetCountry(&infHandle, 0); + std::cout << "Country set to VIETNAM" << std::endl; + + // Step 3: Load engine. + auto startLoad = std::chrono::system_clock::now(); + int loadEngine = LoadANSALPREngineHandle(&infHandle); + auto endLoad = std::chrono::system_clock::now(); + auto loadMs = std::chrono::duration_cast(endLoad - startLoad).count(); + std::cout << "Init Result: " << loadEngine << " (engine load " << loadMs << " ms)" << std::endl; + if (!loadEngine) { + std::cerr << "Failed to load ANSALPR_OCR engine" << std::endl; + ReleaseANSALPRHandle(&infHandle); + return -1; + } + + // Step 4: Enumerate images. + if (!std::filesystem::exists(folderPath) || !std::filesystem::is_directory(folderPath)) { + std::cout << "Folder does not exist or is not a directory: " << folderPath << std::endl; + ReleaseANSALPRHandle(&infHandle); + return -1; + } + + std::vector imageFiles; + const std::set imageExts = { ".jpg", ".jpeg", ".png", ".bmp" }; + for (const auto& entry : std::filesystem::directory_iterator(folderPath)) { + if (!entry.is_regular_file()) continue; + std::string ext = entry.path().extension().string(); + std::transform(ext.begin(), ext.end(), ext.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (imageExts.count(ext)) imageFiles.push_back(entry.path()); + } + std::sort(imageFiles.begin(), imageFiles.end()); + + std::cout << "Scanning folder: " << folderPath << std::endl; + std::cout << "Found " << imageFiles.size() << " image(s)." << std::endl; + + // Step 5: Run inference per image. + int totalDetections = 0; + long long totalInferMs = 0; + for (size_t i = 0; i < imageFiles.size(); ++i) { + const auto& filePath = imageFiles[i]; + cv::Mat input = cv::imread(filePath.string(), cv::IMREAD_COLOR); + if (input.empty()) { + std::cout << "[" << (i + 1) << "/" << imageFiles.size() << "] " + << filePath.filename().string() << " " << std::endl; + continue; + } + + std::string lpnResult; + std::string jpegImage; + cv::Mat* image = new cv::Mat(input); + + auto t0 = std::chrono::system_clock::now(); + ANSALPR_RunInferenceComplete_CPP(&infHandle, &image, "MyCam", 0, 0, lpnResult, jpegImage); + auto t1 = std::chrono::system_clock::now(); + auto inferMs = std::chrono::duration_cast(t1 - t0).count(); + totalInferMs += inferMs; + + std::cout << "\n[" << (i + 1) << "/" << imageFiles.size() << "] " + << filePath.filename().string() + << " (" << input.cols << "x" << input.rows << ")" + << " infer=" << inferMs << "ms" << std::endl; + std::cout << " Raw JSON: " << lpnResult << std::endl; + + int numDetections = 0; + if (!lpnResult.empty()) { + try { + pt.clear(); + std::stringstream ss; + ss << lpnResult; + boost::property_tree::read_json(ss, pt); + auto resultsChild = pt.get_child_optional("results"); + if (resultsChild) { + for (const auto& child : *resultsChild) { + const auto& r = child.second; + const auto class_id = GetData(r, "class_id"); + const auto class_name = GetData(r, "class_name"); + const auto x = GetData(r, "x"); + const auto y = GetData(r, "y"); + const auto width = GetData(r, "width"); + const auto height = GetData(r, "height"); + std::cout << " det[" << numDetections << "]" + << " id=" << class_id + << " name=" << class_name + << " bbox=(" << x << "," << y << "," + << width << "," << height << ")" + << std::endl; + ++numDetections; + } + } + } catch (const std::exception& e) { + std::cout << " JSON parse error: " << e.what() << std::endl; + } + } + std::cout << " => " << numDetections << " detection(s)" << std::endl; + totalDetections += numDetections; + delete image; + } + + // Step 6: Summary. + std::cout << "\n=== ALPR_OCR_FolderInferences_Test summary ===" << std::endl; + std::cout << "Images processed : " << imageFiles.size() << std::endl; + std::cout << "Total detections : " << totalDetections << std::endl; + if (!imageFiles.empty()) { + std::cout << "Avg inference : " + << (totalInferMs / static_cast(imageFiles.size())) << " ms" + << std::endl; + } + + ReleaseANSALPRHandle(&infHandle); + std::cout << "=== ALPR_OCR_FolderInferences_Test complete ===" << std::endl; + return 0; +} + int main() { #ifdef WIN32 @@ -4095,6 +5183,10 @@ int main() #endif // ANSLPR_OD_INDOInferences_FileTest(); //ANSLPR_OD_Inferences_FileTest(); + // Disabled — has a hardcoded NVIDIA RTX 4070 TensorRT-optimised zip path + // (ServerOptimised\ANS_ALPR_v1.2_NVIDIAGeForceRTX4070LaptopGPU.zip) that + // doesn't exist on AMD/Intel machines. Re-enable after fixing the path + // at line 573 if you need the OD folder test. //ANSLPR_OD_FolderInferences_Test(); //ANSLPR_OD_VideoTest(); //ANSLPR_BigSize_VideoTest(); @@ -4109,7 +5201,9 @@ int main() // ANSLPR_MultiGPU_StressTest_FilePlayer(); //ANSLPR_OD_CPU_VideoTest(); //ALPR_OCR_Test(); - ALPR_OCR_VideoTest(); + //ALPR_OCR_VideoTest(); + //ALPR_OCR_FolderInferences_Test(); + ALPR_Helmet_Parallel_VideoTest(); // ANSVIS sim: ALPR + Helmet on same FilePlayer return 0; }