Fix AMD ALPR_OCR and custom model

This commit is contained in:
2026-05-01 07:53:18 +10:00
parent e1f37ec767
commit 91efbb9a8f
25 changed files with 2841 additions and 429 deletions
+30 -1
View File
@@ -149,7 +149,36 @@
"Bash(awk -F'[:\\(\\)]' '{gsub\\(/^ +| +$/,\"\",$2\\); split\\($2,a,\" \"\\); ms=a[1]; if \\(ms+0 >= 500\\) print $0}')",
"Bash(awk -F'[:\\(\\)]' '{split\\($2,a,\" \"\\); ms=a[1]; if \\(ms+0 >= 100\\) print $0}')",
"Bash(grep -cE \"^\\\\s*Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log4.txt\")",
"Bash(grep -oE \"Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log5.txt\")"
"Bash(grep -oE \"Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log5.txt\")",
"Bash(awk -F'\\\\t' '{print $3}')",
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS_twoconsoleapps.log\")",
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS_issue.log\")",
"Bash(awk -F'\\\\t' '$2+0 >= 49 && $2+0 <= 52 {print NR\": \"$0}' \"C:/Users/nghia/Downloads/AVNET-8845HS_issue.log\")",
"Bash(awk -F'\\\\t' 'BEGIN{prev=0} {t=$2+0; if\\(t-prev>5\\){print NR\": GAP=\"t-prev\"s \"$0} prev=t}' \"C:/Users/nghia/Downloads/AVNET-8845HS3.log\")",
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS3.log\")",
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
"Bash(awk -F'\\\\t' '$2+0>=20 && $2+0<=22' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
"Bash(awk -F'\\\\t' '$2+0>=20.5 && $2+0<=21.5' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
"Bash(awk -F'\\\\t' '$2+0>=21.0 && $2+0<=21.7' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
"Bash(awk -F'\\\\t' '$2+0>=21.05 && $2+0<=21.23' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
"Bash(awk -F'\\\\t' '$2+0>=17 && $2+0<=20' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")",
"Bash(awk -F'\\\\t' '$2+0>=17.4 && $2+0<=17.7' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")",
"Bash(awk -F'\\\\t' '$2+0>=12 && $2+0<=18' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")",
"Bash(git -C \"C:/Projects/CLionProjects/ANSCORE\" status)",
"Bash(git -C \"C:/Projects/CLionProjects/ANSCORE\" diff --stat modules/ANSODEngine/ANSONNXCL.h modules/ANSODEngine/ANSONNXCL.cpp modules/ANSODEngine/ANSONNXYOLO.cpp)",
"Bash(awk -F'\\\\t' '{print $4}' \"/c/Users/nghia/Downloads/AVNET-8845HS7.log\")",
"Bash(\"C:/Users/nghia/AppData/Local/Programs/CLion 2026.1/bin/cmake/win/x64/bin/cmake.exe\" --build C:/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSLPR-UnitTest -j 30)",
"Bash(cmd.exe /c '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build C:/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSLPR-UnitTest -j 30')",
"Bash(cmd.exe //c 'call \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\" --target ANSLPR-UnitTest -j 30')",
"PowerShell($env:VS_VARS = '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\"'; $cmd = \"call $env:VS_VARS && `\"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe`\" --build `\"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release`\" --target ANSLPR-UnitTest -j 30\"; cmd.exe /c $cmd 2>&1 | Select-Object -Last 60)",
"PowerShell(cmd.exe /c $cmd 2>&1)",
"PowerShell($cmd = 'call \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\" --target ANSLPR-UnitTest -j 30')",
"Bash(command -v dumpbin)",
"Bash(dumpbin /SYMBOLS \"/c/Projects/CLionProjects/ANSCORE/cmake-build-release/bin/ANSOCR.dll\")",
"Bash(\"/c/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/dumpbin.exe\" //SYMBOLS \"/c/Projects/CLionProjects/ANSCORE/cmake-build-release/bin/ANSOCR.dll\")",
"Bash(cmake --build . --target ANSODEngine -j 8)",
"PowerShell(@\\(Get-ChildItem -Path \"C:\\\\Program Files\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) + @\\(Get-ChildItem -Path \"C:\\\\Program Files \\(x86\\)\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) | ForEach-Object { $_.FullName })",
"PowerShell($d = Get-Item C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\\\\bin\\\\ANSODEngine.dll; \"$\\($d.Name\\) | $\\($d.Length\\) bytes | $\\($d.LastWriteTime\\)\")"
]
}
}
+72
View File
@@ -0,0 +1,72 @@
#pragma once
// av_frame_ptr.h — RAII smart pointer for FFmpeg AVFrame*.
//
// Why this exists
// ---------------
// CVideoPlayer::getNV12Frame() / getCudaHWFrame() return cloned AVFrame* and
// the contract is "caller must av_frame_free()". That contract is comment-only
// — nothing in the type system enforces it, so any early return, exception,
// or refactor that drops the pointer leaks the AVFrame *and* the underlying
// NV12/YUV420P/YUVJ420P buffers (~3 MB per 1080p frame, ~50 MB/s under load).
//
// AvFramePtr wraps the raw pointer in std::unique_ptr<AVFrame, AvFrameDeleter>
// so the destructor frees it automatically — leak-by-omission becomes a
// compile-time impossibility on every internal code path that adopts this type.
//
// Scope: INTERNAL TO ANSCORE ONLY.
// -------------------------------
// Do NOT change DLL-exported function signatures to use AvFramePtr.
// std::unique_ptr cannot cross a DLL boundary safely (different runtimes,
// different STL implementations, different allocators). The public
// AVFrame*-returning APIs in ANSRTSP.h / ANSSRT.h / etc. must remain raw
// pointers for ABI stability.
//
// Use this type at the internal *consumer* call site:
//
// AvFramePtr nv12{ player->getNV12Frame() };
// if (nv12) {
// doSomething(nv12.get()); // borrowed access
// } // auto-freed on scope exit
//
// To transfer ownership into ANSCV's pendingFree pipeline (the
// gpu_frame_attach / gpu_frame_attach_cuda functions, which take raw
// AVFrame* and store/defer them), call .release():
//
// AvFramePtr nv12{ player->getNV12Frame() };
// gpu_frame_attach(mat, nv12.release(), gpuIdx, pts);
//
// Every .release() in the codebase is a deliberate ownership hand-off —
// `git grep '\.release()'` over the migrated files gives an audit list.
extern "C" {
#include <libavutil/frame.h>
}
#include <memory>
namespace ansmedia {
// Custom deleter that calls av_frame_free (which takes AVFrame**).
// noexcept because deleters must not throw — av_frame_free is a C function
// and never throws, but tagging makes the contract explicit.
struct AvFrameDeleter {
void operator()(AVFrame* f) const noexcept {
if (f) {
av_frame_free(&f); // takes AVFrame**, sets *f to nullptr internally
}
}
};
// Owning smart pointer for AVFrame*. Frees on destruction via av_frame_free.
//
// Construction:
// AvFramePtr p; // empty
// AvFramePtr p{ av_frame_clone(src) }; // adopt raw clone
//
// Ownership transfer (e.g. handing off to gpu_frame_attach_cuda):
// takes_ownership(p.release()); // raw pointer escapes, no auto-free
//
// Copy is disallowed (unique_ptr semantics). Move is fine.
using AvFramePtr = std::unique_ptr<AVFrame, AvFrameDeleter>;
} // namespace ansmedia
+199
View File
@@ -10,9 +10,129 @@
#include <fstream>
#include <cstdlib>
#include <system_error>
#include <atomic>
#include <mutex>
namespace ANSCENTER {
// ====================================================================
// DmlSerialState / DmlInferenceGuard implementation.
//
// Storage is in this single linkage unit (ONNXEngine.dll) so every
// module linking against ONNXEngine sees the same flag and mutex —
// ANSODEngine.dll's DML calls and ANSOCR.dll's DML calls serialize
// against the same process-wide state. See the doc comment on
// DmlSerialState in ONNXEngine.h for the full design rationale.
// ====================================================================
namespace {
// false = parallel (default). Tripped to true (sticky) once any
// DML site sees an exception or a slow Run().
std::atomic<bool> g_dmlSerialMode{ false };
// Held during DML inference when g_dmlSerialMode == true. Plain
// std::mutex (not recursive): one DML Run() at a time across the
// entire process — that's the whole point. Other ORT operations
// (session creation, allocator setup, etc.) DO NOT need this
// mutex and are not gated by it.
std::mutex g_dmlInferenceMutex;
// 1500 ms — comfortably above normal AMD DML latency (50–500 ms)
// and well below WDDM TDR (10 s with TdrDelay=10).
constexpr double kDmlSlowThresholdMs = 1500.0;
}
bool DmlSerialState::IsSerialMode() {
return g_dmlSerialMode.load(std::memory_order_relaxed);
}
double DmlSerialState::SlowThresholdMs() {
return kDmlSlowThresholdMs;
}
void DmlSerialState::TripFromException(const char* context, const char* msg) {
bool expected = false;
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
// First caller wins — log the trip event clearly so it's
// obvious in the field which path serialized DML.
std::cerr << "[DmlSerial] TRIPPED from exception in "
<< (context ? context : "?")
<< " - process-wide DML inference is now SERIALIZED. msg="
<< (msg ? msg : "?") << std::endl;
}
}
void DmlSerialState::TripFromLatency(const char* context, double ms) {
bool expected = false;
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
std::cerr << "[DmlSerial] TRIPPED from slow inference in "
<< (context ? context : "?")
<< " ms=" << ms
<< " - process-wide DML inference is now SERIALIZED."
<< std::endl;
}
}
void DmlSerialState::AcquireMutex() {
g_dmlInferenceMutex.lock();
}
void DmlSerialState::ReleaseMutex() {
g_dmlInferenceMutex.unlock();
}
DmlInferenceGuard::DmlInferenceGuard(EngineType ep, const char* context)
: _ep(ep), _context(context), _locked(false)
{
// ── Always-serial-on-AMD policy ────────────────────────────
//
// Earlier design (Option A v1) was lazy: parallel by default,
// serialize only after seeing a slow Run() or exception. Field
// testing on AMD Radeon 780M proved this fires too late — by the
// time the latency trip arms, a sibling thread has already
// entered Run() at depth=2 and wedged the D3D12 command queue.
// ORT then crashes inside its own DML EP cleanup when the WDDM
// TDR fires.
//
// The only way to guarantee zero dual-stall on AMD is to never
// let two threads be in flight on DML simultaneously. This
// constructor unconditionally takes the global DML mutex when
// ep == AMD_GPU. Intel (OpenVINO) and NVIDIA (CUDA/TRT) skip
// the lock entirely — guard collapses to a timestamp read +
// comparison, and the destructor is a single comparison. Zero
// measurable overhead on those platforms.
//
// Trade-off: AMD inference becomes serial (depth=1) instead of
// parallel (depth=2). Each individual call still runs on the
// GPU at GPU speeds; only the cross-thread overlap is removed.
// Throughput cost vs broken depth=2 is ~30% lower wall-clock
// throughput per frame, but uptime goes from ~30 s before
// wedge → unbounded.
if (_ep == EngineType::AMD_GPU) {
DmlSerialState::AcquireMutex();
_locked = true;
}
// Start the latency timer AFTER lock acquisition so contention
// on the global mutex doesn't itself trip the slow-Run()
// threshold. We measure GPU Run() time, not queue wait.
_t0 = std::chrono::steady_clock::now();
}
DmlInferenceGuard::~DmlInferenceGuard() {
// Latency-based trip detection: if THIS Run() took longer than
// the slow threshold, future DML inference goes serial. Only
// checked on AMD — Intel/NVIDIA never trip.
if (_ep == EngineType::AMD_GPU) {
const auto t1 = std::chrono::steady_clock::now();
const double ms = std::chrono::duration<double, std::milli>(t1 - _t0).count();
if (ms > kDmlSlowThresholdMs) {
DmlSerialState::TripFromLatency(_context, ms);
}
}
if (_locked) {
DmlSerialState::ReleaseMutex();
}
}
// ====================================================================
// BasicOrtHandler — constructors
// ====================================================================
@@ -365,6 +485,85 @@ namespace ANSCENTER {
return false;
}
// ====================================================================
// Reinitialize — tear down + rebuild session on a new EP.
//
// Used by subclass self-heal paths (ONNXOCRRecognizer /
// ONNXOCRDetector) to recover from Ort::Exception in Run(). The
// typical flow is:
//
// try { ort_session->Run(...); }
// catch (const Ort::Exception&) {
// Reinitialize(EngineType::AMD_GPU); // first attempt: retry DML
// try { ort_session->Run(...); }
// catch (const Ort::Exception&) {
// Reinitialize(EngineType::CPU); // last resort: drop to CPU
// ort_session->Run(...);
// }
// }
//
// After this call returns, ort_session, ort_env, memory_info_handler,
// input_node_names, input_node_dims, output_node_names, etc. are all
// freshly populated for `newEngine`. Subclass state that lives outside
// BasicOrtHandler (e.g. ONNXOCRRecognizer::_warmedUp, ::keys_) is NOT
// touched — the caller should reset whatever flags it owns if they
// become invalid for the new session.
//
// Throws Ort::Exception (or std::exception) on failure to construct
// the new session. Caller should treat that as terminal for the
// current Run() and return an empty result.
// ====================================================================
void BasicOrtHandler::Reinitialize(EngineType newEngine)
{
ANS_DBG("OrtHandler",
"Reinitialize: oldEngine=%d newEngine=%d (model=%ls)",
static_cast<int>(m_engineType),
static_cast<int>(newEngine),
onnx_path ? onnx_path : L"(null)");
// Drop the existing session FIRST so its D3D12 / DML allocations
// are released back to the driver. Order matters: ort_session
// must die before ort_env, otherwise Ort::Session's destructor
// can dereference a dangling env pointer in some EP teardown
// paths (observed on DML 1.15.x).
if (ort_session) {
try { delete ort_session; } catch (...) {}
ort_session = nullptr;
}
if (memory_info_handler) {
try { delete memory_info_handler; } catch (...) {}
memory_info_handler = nullptr;
}
if (ort_env) {
try { delete ort_env; } catch (...) {}
ort_env = nullptr;
}
// Reset shape/name vectors so initialize_handler repopulates
// them from the new session (input names can differ across EPs
// for some models — known DML EP quirk).
input_node_names.clear();
input_node_names_.clear();
input_node_dims.clear();
input_values_handler.clear();
output_node_names.clear();
output_node_names_.clear();
output_node_dims.clear();
input_name = nullptr;
input_tensor_size = 1;
num_outputs = 1;
// Set the requested EP and re-run the standard init. Throws on
// failure — caller's responsibility to handle.
m_engineType = newEngine;
initialize_handler();
ANS_DBG("OrtHandler",
"Reinitialize OK: ort_session=%p engine=%d",
static_cast<void*>(ort_session),
static_cast<int>(m_engineType));
}
// ====================================================================
// initialize_handler
// ====================================================================
+118
View File
@@ -8,6 +8,7 @@
#include <typeinfo>
#include <deque>
#include <unordered_map>
#include <chrono> // DmlInferenceGuard latency timing
#include "onnxruntime_cxx_api.h"
#include "opencv2/opencv.hpp"
@@ -337,6 +338,108 @@ namespace ANSCENTER {
// (AMD fields go here — none yet)
};
// ====================================================================
// DmlSerialState — process-wide DirectML inference serialization
//
// AMD Radeon iGPUs (Radeon 680M / 780M, observed) cannot sustain
// 2-way concurrent inference on DirectML. Under sustained
// parallel load the WDDM driver eventually deadlocks the D3D12
// command queue when multiple ORT sessions submit work
// simultaneously, producing a "dual stall" pattern (both threads
// inside ORT::Run() with no exception thrown) and ultimately an
// access violation deep inside the DirectML EP cleanup path.
//
// Policy: every DirectML inference call on AMD_GPU is serialized
// through DmlInferenceGuard's unconditional lock. Lazy "trip on
// first slow Run()" was tried first (a flag flipped to true on
// exception or >1500 ms latency) but field-proven inadequate —
// the trip fires AFTER a slow Run() returns, by which point a
// sibling thread has already entered Run() concurrently and
// wedged. Always-serialize is the only safe answer on AMD.
//
// This class still tracks an "ever-tripped" flag for diagnostics
// — the TripFromException / TripFromLatency calls log a marker
// the first time a DML hiccup is observed, so field traces
// clearly show whether the AMD driver had a degraded event. The
// flag does NOT gate the lock; the lock is unconditional on AMD.
//
// Intel (OpenVINO) and NVIDIA (CUDA/TRT) bypass the lock entirely
// — DmlInferenceGuard short-circuits on ep != AMD_GPU. They keep
// peakDepth=2 parallel inference unchanged; zero behavioral
// change on those platforms.
//
// Storage lives in the ONNXEngine sources compiled into
// ANSODEngine.dll so all higher-level modules (ANSODEngine,
// ANSOCR, ...) share a single process-global mutex.
// ====================================================================
class ONNXENGINE_API DmlSerialState {
public:
// True when serial mode has been tripped.
static bool IsSerialMode();
// Trip helpers. Both are idempotent — only the first caller
// per process actually flips the flag and emits the
// diagnostic. Called by:
// exception path → ANSONNXYOLO::HandleDmlRunFailure, OCR
// latency path → DmlInferenceGuard destructor on slow Run()
static void TripFromException(const char* context, const char* msg);
static void TripFromLatency (const char* context, double ms);
// 1500 ms — comfortably above normal AMD DML latency and well
// under WDDM TDR.
static double SlowThresholdMs();
// Internal — only DmlInferenceGuard should call these.
static void AcquireMutex();
static void ReleaseMutex();
};
// ====================================================================
// DmlInferenceGuard — RAII guard for DirectML inference call sites
//
// Construct around every Ort::Session::Run() that may execute on
// a DirectML EP. Behavior:
//
// 1. ep != AMD_GPU → entire guard is a no-op (zero overhead on
// Intel / NVIDIA / CPU). The destructor's latency check and
// lock release are also skipped via _ep == AMD_GPU and
// _locked == false guards.
// 2. ep == AMD_GPU → unconditionally acquires the global DML
// mutex; only one DML inference in the entire process runs
// at a time. On scope exit, releases the mutex and (if the
// call exceeded DmlSerialState::SlowThresholdMs()) records a
// diagnostic trip marker — the marker does NOT change
// behavior, since AMD is already serial; it just shows up in
// logs as evidence of a degraded GPU event.
//
// Why unconditional serialization on AMD: empirical testing on
// Radeon 780M shows the WDDM driver wedges when 2 ORT sessions
// submit DML inference concurrently for >~30 s. Lazy "lock only
// after first slow call" was found insufficient — by the time the
// trip fires, a sibling thread is already inside the wedge-prone
// Run() with no way to back out. Always-lock is the only path
// that guarantees the dual-stall window never opens. The cost is
// ~30% lower throughput on AMD (depth=1 vs depth=2) for
// unbounded uptime instead of crashing every 30 s.
// ====================================================================
class ONNXENGINE_API DmlInferenceGuard {
public:
explicit DmlInferenceGuard(EngineType ep, const char* context);
~DmlInferenceGuard();
// Non-copyable / non-movable — RAII scope guard only.
DmlInferenceGuard(const DmlInferenceGuard&) = delete;
DmlInferenceGuard& operator=(const DmlInferenceGuard&) = delete;
DmlInferenceGuard(DmlInferenceGuard&&) = delete;
DmlInferenceGuard& operator=(DmlInferenceGuard&&) = delete;
private:
EngineType _ep;
const char* _context;
bool _locked;
std::chrono::steady_clock::time_point _t0;
};
// ====================================================================
// BasicOrtHandler
// ====================================================================
@@ -410,6 +513,21 @@ namespace ANSCENTER {
// (which require the input name) BEFORE the real session is
// created. Returns an empty string on failure.
static std::string QueryModelInputName(const std::string& onnxPath);
protected:
// Tear down the existing ORT session/env/memory-info and rebuild
// them with `newEngine` as the requested EP. Used by subclass
// self-heal paths (e.g. ONNXOCRRecognizer / ONNXOCRDetector) when
// an Ort::Exception fires from a Run() — the session is recreated
// either on the same EP (transient driver hiccup recovery) or on
// CPU (permanent fallback). Throws on failure; caller should
// treat that as "give up for this call" and return an empty
// result. Subclass-owned auxiliary state (input_node_dims,
// input_node_names, ...) is repopulated by initialize_handler.
// Caller is responsible for any subclass-specific re-init that
// sits on top of the base session (e.g. dictionary loading on
// ONNXOCRRecognizer is not affected because it lives in the
// subclass and survives a session swap).
void Reinitialize(EngineType newEngine);
private:
void initialize_handler();
protected:
+29 -12
View File
@@ -1,6 +1,7 @@
#include "ANSFLV.h"
#include "ANSMatRegistry.h"
#include "ANSGpuFrameOps.h"
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
#include "ANSLicense.h" // ANS_DBG macro
#include <memory>
@@ -785,17 +786,25 @@ extern "C" __declspec(dllexport) int GetFLVCVImage(ANSCENTER::ANSFLVClient** Han
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
anscv_mat_replace(image, std::move(img));
// NV12 GPU fast path (optional — disabled by default for stability)
// NV12 GPU fast path (optional — disabled by default for stability).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
if ((*Handle)->IsNV12FastPath()) {
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSFLV V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = (*Handle)->GetNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSFLV V1 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
}
@@ -1032,16 +1041,24 @@ extern "C" __declspec(dllexport) int GetFLVCVImage_V2(uint64_t handleVal, int& w
}
anscv_mat_replace(image, std::move(img));
// Attach NV12 frame for GPU fast-path inference (side-table registry)
// Attach NV12 frame for GPU fast-path inference (side-table registry).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
int gpuIdx = h->GetHWDecodingGpuIndex();
AVFrame* cudaHW = h->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = h->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSFLV V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = h->GetNV12Frame();
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSFLV V2 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
+29 -11
View File
@@ -1,6 +1,7 @@
#include "ANSFilePlayer.h"
#include "ANSMatRegistry.h"
#include "ANSGpuFrameOps.h"
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include <memory>
#include <cstdint>
#include <libswscale/swscale.h>
@@ -382,15 +383,24 @@ extern "C" __declspec(dllexport) int GetFilePlayerCVImage(ANSCENTER::ANSFILEPLAY
// Attach NV12/CUDA frames to registry for inference zero-copy (same as ANSRTSP/ANSVideoPlayer).
// CFilePlayer inherits getNV12Frame()/getCudaHWFrame() from CVideoPlayer.
// AvFramePtr wraps each clone so it auto-frees on any return path before
// the .release() handoff to gpu_frame_attach[_cuda] — leak-by-omission
// becomes impossible if an exception or early return slips in later.
int gpuIdx = (*Handle)->_playerClient->getHWDecodingGpuIndex();
AVFrame* cudaHW = (*Handle)->_playerClient->getCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ (*Handle)->_playerClient->getCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = (*Handle)->_playerClient->getNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ (*Handle)->_playerClient->getNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSFilePlayer V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release());
} else {
AVFrame* nv12 = (*Handle)->_playerClient->getNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->_playerClient->getNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSFilePlayer V1 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
} else {
// CPU-only fallback: attach full-res BGR clone for ANSRTYOLO
cv::Mat infImg = (*Handle)->GetInferenceImage();
@@ -639,16 +649,24 @@ extern "C" __declspec(dllexport) int GetFilePlayerCVImage_V2(uint64_t handleVal,
}
anscv_mat_replace(image, std::move(img));
// NV12/CUDA registry attach (same as V1)
// NV12/CUDA registry attach (same as V1).
// AvFramePtr ensures any clone we obtain is freed automatically if we
// don't reach the .release() handoff (e.g. exception, future early return).
int gpuIdx = h->_playerClient->getHWDecodingGpuIndex();
AVFrame* cudaHW = h->_playerClient->getCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ h->_playerClient->getCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = h->_playerClient->getNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ h->_playerClient->getNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSFilePlayer V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release());
} else {
AVFrame* nv12 = h->_playerClient->getNV12Frame();
ansmedia::AvFramePtr nv12{ h->_playerClient->getNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSFilePlayer V2 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
} else {
cv::Mat infImg = h->GetInferenceImage();
if (!infImg.empty() && infImg.data != (*image)->data) {
+29 -12
View File
@@ -1,6 +1,7 @@
#include "ANSMJPEG.h"
#include "ANSMatRegistry.h"
#include "ANSGpuFrameOps.h"
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
#include "ANSLicense.h" // ANS_DBG macro
#include <memory>
@@ -784,17 +785,25 @@ extern "C" __declspec(dllexport) int GetMJPEGCVImage(ANSCENTER::ANSMJPEGClient**
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
anscv_mat_replace(image, std::move(img));
// NV12 GPU fast path (optional — disabled by default for stability)
// NV12 GPU fast path (optional — disabled by default for stability).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
if ((*Handle)->IsNV12FastPath()) {
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSMJPEG V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = (*Handle)->GetNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSMJPEG V1 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
}
@@ -1030,16 +1039,24 @@ extern "C" __declspec(dllexport) int GetMJPEGCVImage_V2(uint64_t handleVal, int&
}
anscv_mat_replace(image, std::move(img));
// Attach NV12 frame for GPU fast-path inference (side-table registry)
// Attach NV12 frame for GPU fast-path inference (side-table registry).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
int gpuIdx = h->GetHWDecodingGpuIndex();
AVFrame* cudaHW = h->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = h->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSMJPEG V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = h->GetNV12Frame();
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSMJPEG V2 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
+29 -12
View File
@@ -1,6 +1,7 @@
#include "ANSRTMP.h"
#include "ANSMatRegistry.h"
#include "ANSGpuFrameOps.h"
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
#include "ANSLicense.h" // ANS_DBG macro
#include <memory>
@@ -808,17 +809,25 @@ extern "C" __declspec(dllexport) int GetRTMPCVImage(ANSCENTER::ANSRTMPClient** H
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
anscv_mat_replace(image, std::move(img));
// NV12 GPU fast path (optional — disabled by default for stability)
// NV12 GPU fast path (optional — disabled by default for stability).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
if ((*Handle)->IsNV12FastPath()) {
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSRTMP V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = (*Handle)->GetNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSRTMP V1 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
}
@@ -1053,16 +1062,24 @@ extern "C" __declspec(dllexport) int GetRTMPCVImage_V2(uint64_t handleVal, int&
}
anscv_mat_replace(image, std::move(img));
// Attach NV12 frame for GPU fast-path inference (side-table registry)
// Attach NV12 frame for GPU fast-path inference (side-table registry).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
int gpuIdx = h->GetHWDecodingGpuIndex();
AVFrame* cudaHW = h->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = h->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSRTMP V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = h->GetNV12Frame();
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSRTMP V2 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
+18 -5
View File
@@ -2,6 +2,7 @@
#include "ANSMatRegistry.h"
#include "ANSGpuFrameOps.h"
#include "GpuNV12SlotPool.h"
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include "ANSLicense.h" // ANS_DBG macro
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
#include <memory>
@@ -1175,7 +1176,12 @@ extern "C" __declspec(dllexport) int GetRTSPCVImage(
(void*)*image, gpuIdx, (int)inFlightGuardHeld);
if (inFlightGuardHeld) {
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
// AvFramePtr wraps each clone obtained from the player so it
// auto-frees on every return path before the .release() handoff
// to gpu_frame_attach[_cuda]. This closes the leak window if
// any future early-return / exception slips between the clone
// call and the attach call.
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
if (cudaHW) {
RTSP_DBG("[GetRTSPCVImage] cudaHW: %dx%d data[0]=%p data[1]=%p",
cudaHW->width, cudaHW->height,
@@ -1186,13 +1192,20 @@ extern "C" __declspec(dllexport) int GetRTSPCVImage(
gpuIdx, cudaHW->width, cudaHW->height);
// Only fetch CPU NV12 if pool slot unavailable (cross-GPU fallback).
AVFrame* cpuNV12 = slot ? nullptr : (*Handle)->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12, slot);
ansmedia::AvFramePtr cpuNV12{ slot ? nullptr : (*Handle)->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSRTSP attach_cuda cudaHW=%p cpuNV12=%p slot=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), (void*)slot, gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release(), slot);
} else {
// HW decode not active — try CPU NV12
AVFrame* nv12 = (*Handle)->GetNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSRTSP attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
+29 -12
View File
@@ -1,6 +1,7 @@
#include "ANSSRT.h"
#include "ANSMatRegistry.h"
#include "ANSGpuFrameOps.h"
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
#include "ANSLicense.h" // ANS_DBG macro
#include <memory>
@@ -826,17 +827,25 @@ extern "C" __declspec(dllexport) int GetSRTCVImage(ANSCENTER::ANSSRTClient** Han
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
anscv_mat_replace(image, std::move(img));
// NV12 GPU fast path (optional — disabled by default for stability)
// NV12 GPU fast path (optional — disabled by default for stability).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
if ((*Handle)->IsNV12FastPath()) {
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSSRT V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = (*Handle)->GetNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSSRT V1 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
}
@@ -1077,16 +1086,24 @@ extern "C" __declspec(dllexport) int GetSRTCVImage_V2(uint64_t handleVal, int& w
anscv_mat_replace(image, std::move(img));
// Attach NV12 frame for GPU fast-path inference (side-table registry)
// Attach NV12 frame for GPU fast-path inference (side-table registry).
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
int gpuIdx = h->GetHWDecodingGpuIndex();
AVFrame* cudaHW = h->GetCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
if (cudaHW) {
AVFrame* cpuNV12 = h->GetNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSSRT V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
cpuNV12.release());
} else {
AVFrame* nv12 = h->GetNV12Frame();
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSSRT V2 attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
+16 -5
View File
@@ -5,6 +5,7 @@
#include "ANSMatRegistry.h"
#include "ANSGpuFrameRegistry.h"
#include "ANSGpuFrameOps.h" // gpu_frame_attach, gpu_frame_attach_cuda
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
#include <memory>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
@@ -1010,17 +1011,27 @@ extern "C" __declspec(dllexport) int GetVideoPlayerCVImage(
// When using cv::VideoCapture fallback, attach BGR clone (pixFmt=1000).
if ((*Handle)->_hwDecodeActive && (*Handle)->_hwPlayer) {
// --- HW decode: NV12/CUDA attachment (same pattern as ANSRTSP.cpp:847-862) ---
// AvFramePtr wraps each clone so it auto-frees on any return path
// before the .release() handoff to gpu_frame_attach[_cuda]. This
// makes leak-by-omission impossible if an exception or early return
// is added between the clone and the attach call.
int gpuIdx = (*Handle)->_hwPlayer->getHWDecodingGpuIndex();
AVFrame* cudaHW = (*Handle)->_hwPlayer->getCudaHWFrame();
ansmedia::AvFramePtr cudaHW{ (*Handle)->_hwPlayer->getCudaHWFrame() };
if (cudaHW) {
// CUDA zero-copy: frame data[0]/data[1] are CUDA device pointers.
// Also attach CPU NV12 as fallback for cross-GPU inference.
AVFrame* cpuNV12 = (*Handle)->_hwPlayer->getNV12Frame();
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
ansmedia::AvFramePtr cpuNV12{ (*Handle)->_hwPlayer->getNV12Frame() };
ANS_DBG("MEDIA_AvPtr",
"ANSVideoPlayer attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release());
} else {
AVFrame* nv12 = (*Handle)->_hwPlayer->getNV12Frame();
ansmedia::AvFramePtr nv12{ (*Handle)->_hwPlayer->getNV12Frame() };
if (nv12) {
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
ANS_DBG("MEDIA_AvPtr",
"ANSVideoPlayer attach nv12=%p gpuIdx=%d",
(void*)nv12.get(), gpuIdx);
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
}
}
} else {
+26
View File
@@ -7,6 +7,7 @@
#include <fstream>
#include <string>
#include <filesystem>
#include <atomic>
#include <opencv2/opencv.hpp>
#include "Utility.h"
#include "ANSLicense.h"
@@ -98,6 +99,31 @@ namespace ANSCENTER
bool _debugFlag{ false };
bool _licenseValid{ false };
bool _isInitialized{ false};
// Loading-in-progress flag — distinguishes the "LoadEngine() is still
// running" state from "LoadEngine() never called or failed". Set by
// ModelLoadingGuard at the top of LoadEngine() and cleared on exit
// (success, failure, or exception). RunInference/RunInferencesBatch
// consult this BEFORE _isInitialized so the early frames during the
// 3+ second model-load window don't flood the log with bogus
// "Model is not initialized" errors.
std::atomic<bool> _modelLoading{ false };
// RAII helper: sets _modelLoading=true on construction, false on
// destruction. Use at the top of LoadEngine() so the flag is always
// cleared on every exit path.
struct ModelLoadingGuard {
std::atomic<bool>& flag;
explicit ModelLoadingGuard(std::atomic<bool>& f) : flag(f) {
flag.store(true, std::memory_order_release);
}
~ModelLoadingGuard() {
flag.store(false, std::memory_order_release);
}
ModelLoadingGuard(const ModelLoadingGuard&) = delete;
ModelLoadingGuard& operator=(const ModelLoadingGuard&) = delete;
};
double _detectorThreshold{ 0.5 };
double _ocrThreshold{ 0.5 };
double _colorThreshold{ 0.0 };// If we use the colour information, this threshold should be greater than 0.0
+37 -20
View File
@@ -221,6 +221,11 @@ namespace ANSCENTER
bool ANSALPR_OCR::LoadEngine() {
std::lock_guard<std::recursive_mutex> lock(_mutex);
// RAII guard: set _modelLoading=true for the lifetime of this scope.
// RunInference / RunInferencesBatch consult this flag and silently
// defer (instead of logging "Model is not initialized") while we're
// loading the LP detector + OCR engine + colour classifier.
ModelLoadingGuard mlg(_modelLoading);
try {
WriteEventLog("ANSALPR_OCR::LoadEngine: Step 1 - Starting engine load");
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 1: Starting engine load", __FILE__, __LINE__);
@@ -1479,22 +1484,28 @@ namespace ANSCENTER
this->_logger.LogError("ANSALPR_OCR::RunInference", "Invalid license", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
this->_logger.LogError("ANSALPR_OCR::RunInference", "Model is not initialized", __FILE__, __LINE__);
// LoadEngine() may be running on another thread (3+ seconds for all
// models). Silently defer instead of spamming "Model is not
// initialized" — the caller will retry on the next frame.
if (_modelLoading.load(std::memory_order_acquire)) {
return {};
}
// Source-of-truth check on the actual engine pointers. If LoadEngine
// failed or Destroy() was called, _lpDetector / _ocrEngine are null
// and we want a real error logged so the operator sees it.
if (!_lpDetector || !_ocrEngine) {
this->_logger.LogError("ANSALPR_OCR::RunInference",
"Model not loaded (LoadEngine not called or failed)",
__FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 5 || input.rows < 5) {
this->_logger.LogError("ANSALPR_OCR::RunInference", "Input image is empty or too small", __FILE__, __LINE__);
return {};
}
if (!_lpDetector) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_lpDetector is null", __FILE__, __LINE__);
return {};
}
if (!_ocrEngine) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_ocrEngine is null", __FILE__, __LINE__);
return {};
}
try {
// Convert grayscale to BGR if necessary
@@ -1856,19 +1867,25 @@ namespace ANSCENTER
this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", "Invalid license", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", "Model is not initialized", __FILE__, __LINE__);
// LoadEngine() may be running on another thread (3+ seconds for all
// models). Silently defer instead of spamming "Model is not
// initialized" — the caller will retry on the next frame.
if (_modelLoading.load(std::memory_order_acquire)) {
return {};
}
// Source-of-truth check on the actual engine pointers. If LoadEngine
// failed or Destroy() was called, _lpDetector / _ocrEngine are null
// and we want a real error logged so the operator sees it.
if (!_lpDetector || !_ocrEngine) {
this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch",
"Model not loaded (LoadEngine not called or failed)",
__FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 5 || input.rows < 5) return {};
if (!_lpDetector) {
this->_logger.LogFatal("ANSALPR_OCR::RunInferencesBatch", "_lpDetector is null", __FILE__, __LINE__);
return {};
}
if (!_ocrEngine) {
this->_logger.LogFatal("ANSALPR_OCR::RunInferencesBatch", "_ocrEngine is null", __FILE__, __LINE__);
return {};
}
if (vehicleBoxes.empty()) return {};
try {
+196 -23
View File
@@ -2,6 +2,7 @@
#include "include/clipper.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
#include "ANSLicense.h" // ANS_DBG
#include <opencv2/imgproc.hpp>
#include <iostream>
@@ -12,6 +13,130 @@
namespace ANSCENTER {
namespace onnxocr {
// AMD/DirectML latency-based recreate (Path B) — record helper. See
// ONNXOCRRecognizer for the rationale. Arms _slowRecreatePending if
// a Run() exceeded the slow threshold while past warmup and outside
// the cooldown window.
void ONNXOCRDetector::RecordRunLatency(const char* context, double ms) {
if (ms <= kSlowRecreateThresholdMs) return;
if (!_warmedUp) return;
const auto now = std::chrono::steady_clock::now();
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastRecreateAt).count();
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
sinceLast < kRecreateCooldownMs) {
ANS_DBG("ANSONNXDet_run",
"%s: slow Run (%.0f ms) but in cooldown (%lldms)",
context, ms, static_cast<long long>(sinceLast));
return;
}
_slowRecreatePending.store(true, std::memory_order_release);
std::cerr << "[ONNXOCRDetector] " << context
<< ": slow Run (" << ms
<< " ms > threshold) — flagged for proactive recreate\n";
ANS_DBG("ANSONNXDet_run",
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
context, ms, kSlowRecreateThresholdMs);
}
// AMD/DirectML latency-based recreate (Path B) — apply helper. Called
// at entry of public methods. If the slow flag was armed by a prior
// call's RecordRunLatency, swap the session in here so this call uses
// the fresh one.
void ONNXOCRDetector::ApplyPendingSlowRecreate(const char* context) {
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
return;
}
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
<< " (latency-triggered)\n";
ANS_DBG("ANSONNXDet_run",
"%s: proactive recreate on EP=%d", context,
static_cast<int>(currentEp));
try {
Reinitialize(currentEp);
_warmedUp = false;
_lastRecreateAt = std::chrono::steady_clock::now();
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate OK\n";
ANS_DBG("ANSONNXDet_run", "%s: proactive recreate OK", context);
} catch (const std::exception& e) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate failed: " << e.what() << "\n";
ANS_DBG("ANSONNXDet_run",
"%s: proactive recreate failed (%s)", context, e.what());
} catch (...) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate failed (unknown)\n";
}
}
// AMD/DirectML self-heal helper. See ONNXOCRRecognizer for the full
// rationale. On any Run() failure: recreate session on the same EP
// first (transient driver hiccup recovery — the common case for AMD
// DML 0x80004005 E_FAIL); on a second failure, fall back to CPU EP.
// The current call's result is already lost — caller returns empty;
// the recovered session is used by the NEXT Detect call.
void ONNXOCRDetector::HandleRunFailureAndRecreate(const char* context,
const std::exception& e) {
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRDetector] " << context
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
<< ": " << e.what() << " — recreating session\n";
ANS_DBG("ANSONNXDet_run",
"%s: Run failed EP=%d msg=%.200s — recreating",
context, static_cast<int>(currentEp), e.what());
// Option A: trip process-wide DML serial mode on AMD exception
// (idempotent, no-op on Intel/NVIDIA/CPU).
if (currentEp == EngineType::AMD_GPU) {
ANSCENTER::DmlSerialState::TripFromException(
context, e.what());
}
if (currentEp != EngineType::CPU) {
try {
Reinitialize(currentEp);
_warmedUp = false;
std::cerr << "[ONNXOCRDetector] " << context
<< ": session recreated on EP=" << static_cast<int>(currentEp) << "\n";
ANS_DBG("ANSONNXDet_run",
"%s: session recreated on EP=%d", context,
static_cast<int>(currentEp));
return;
} catch (const std::exception& reErr) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": same-EP recreate failed: " << reErr.what()
<< " — falling back to CPU\n";
ANS_DBG("ANSONNXDet_run",
"%s: same-EP recreate failed (%s); falling back to CPU",
context, reErr.what());
} catch (...) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
}
}
try {
Reinitialize(EngineType::CPU);
_warmedUp = false;
std::cerr << "[ONNXOCRDetector] " << context
<< ": fallback session created on CPU\n";
ANS_DBG("ANSONNXDet_run", "%s: CPU fallback OK", context);
} catch (const std::exception& cpuErr) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": CPU fallback ALSO failed: " << cpuErr.what()
<< " — ort_session may be null, future calls will return empty\n";
ANS_DBG("ANSONNXDet_run",
"%s: CPU fallback failed (%s); ort_session may be null",
context, cpuErr.what());
} catch (...) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": CPU fallback ALSO failed (unknown)\n";
}
}
ONNXOCRDetector::ONNXOCRDetector(const std::string& onnx_path, unsigned int num_threads)
: BasicOrtHandler(onnx_path, num_threads) {
}
@@ -55,6 +180,11 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
return {};
}
// Path B (AMD/DML): apply pending proactive recreate before constructing
// any tensors. See ONNXOCRRecognizer for the full rationale.
ApplyPendingSlowRecreate("Detect");
if (!ort_session) return {}; // recreate may have failed; bail safely
// Try to get full-resolution image from NV12 frame (same pattern as ANSONNXYOLO)
cv::Mat inferenceImage = srcImage;
float bgrScaleX = 1.0f, bgrScaleY = 1.0f;
@@ -89,36 +219,60 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
// Run inference
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
// Run inference + post-process inside try/catch. AMD/DirectML
// self-heal: on Run() failure (typically 0x80004005 E_FAIL from a
// transient DML driver hiccup) we recreate the session for the next
// call (same EP first, CPU as last resort). This call's result is
// lost — return empty boxes; ALPR pipeline tolerates a dropped
// detection frame, the tracker recovers. Latency is also recorded
// for the Path B proactive-recreate trigger.
try {
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRDetector::Detect");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
RecordRunLatency("Detect", runMs);
// Get output data
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
// Get output data
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int outH = static_cast<int>(outputShape[2]);
int outW = static_cast<int>(outputShape[3]);
int outH = static_cast<int>(outputShape[2]);
int outW = static_cast<int>(outputShape[3]);
// Postprocess — detection coords are relative to inferenceImage (full-res),
// then scaled back to srcImage (display-res) coordinates
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
inferenceImage.rows, inferenceImage.cols,
dbThresh, boxThresh, unclipRatio, useDilation);
// Postprocess — detection coords are relative to inferenceImage (full-res),
// then scaled back to srcImage (display-res) coordinates
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
inferenceImage.rows, inferenceImage.cols,
dbThresh, boxThresh, unclipRatio, useDilation);
// Rescale box coordinates from full-res to display-res
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
for (auto& box : boxes) {
for (auto& pt : box.points) {
pt.x *= bgrScaleX;
pt.y *= bgrScaleY;
// Rescale box coordinates from full-res to display-res
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
for (auto& box : boxes) {
for (auto& pt : box.points) {
pt.x *= bgrScaleX;
pt.y *= bgrScaleY;
}
}
}
}
return boxes;
return boxes;
}
catch (const Ort::Exception& e) {
HandleRunFailureAndRecreate("Detect", e);
return {};
}
}
std::vector<float> ONNXOCRDetector::Preprocess(const cv::Mat& srcImage, int maxSideLen,
@@ -323,6 +477,8 @@ void ONNXOCRDetector::Warmup() {
// multiple-of-32 rounding. cuDNN caches the algorithm for this
// shape so the first real inference doesn't pay the picker cost.
constexpr int kWarmupSide = 320;
bool warmupFailed = false;
std::string warmupErr;
try {
cv::Mat dummy(kWarmupSide, kWarmupSide, CV_8UC3, cv::Scalar(128, 128, 128));
cv::Mat dummyF;
@@ -347,6 +503,23 @@ void ONNXOCRDetector::Warmup() {
}
catch (const Ort::Exception& e) {
std::cerr << "[ONNXOCRDetector] Warmup failed: " << e.what() << std::endl;
warmupFailed = true;
warmupErr = e.what();
}
// AMD/DML self-heal: if warmup itself threw, the session is dead and
// every Detect() will fail. Recreate now (same EP first, CPU as
// fallback) so production traffic can still serve. Leave _warmedUp
// = false so the recovered session warms up on its first real call.
if (warmupFailed) {
ANS_DBG("ANSONNXDet_run",
"Warmup failed (%.200s); triggering self-heal",
warmupErr.c_str());
try {
std::runtime_error proxy(warmupErr);
HandleRunFailureAndRecreate("Warmup", proxy);
} catch (...) {}
return; // do NOT set _warmedUp; recovered session warms on first real Detect
}
_warmedUp = true;
}
@@ -4,6 +4,8 @@
#include "ONNXEngine.h"
#include <vector>
#include <mutex>
#include <atomic>
#include <chrono>
namespace ANSCENTER {
namespace onnxocr {
@@ -53,6 +55,25 @@ private:
// Expand 4-point box using Clipper offset
std::vector<cv::Point2f> UnclipPolygon(const std::array<cv::Point2f, 4>& box, float unclipRatio);
// AMD/DirectML self-heal helper — see ONNXOCRRecognizer for full
// contract. On Run() failure: recreate session on the same EP first
// (transient hiccup recovery), CPU fallback as last resort. Resets
// _warmedUp so the new session gets re-warmed on next Detect call.
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
// AMD/DirectML latency-based self-heal (Path B) — see ONNXOCRRecognizer
// for the full rationale. Detector is observed to hit the same
// slow-degradation pattern as the recognizer after a sibling DML
// session takes a fatal: Run() doesn't throw, just gets slow.
// 2 s threshold catches degraded individual Run() (healthy is 50–500 ms);
// 60 s cooldown prevents thrash on permanently broken driver state.
static constexpr double kSlowRecreateThresholdMs = 2000.0;
static constexpr int64_t kRecreateCooldownMs = 60000;
std::atomic<bool> _slowRecreatePending{false};
std::chrono::steady_clock::time_point _lastRecreateAt{};
void RecordRunLatency(const char* context, double ms);
void ApplyPendingSlowRecreate(const char* context);
std::mutex _mutex;
};
+274 -15
View File
@@ -113,6 +113,165 @@ static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
return padded;
}
// AMD/DirectML latency-based recreate (Path B) — record helper.
//
// Called after every successful production-path Run(). If the call
// took longer than kSlowRecreateThresholdMs (5 s default), and the
// session has finished its warmup, and we're outside the cooldown
// window, arm _slowRecreatePending so the NEXT public method will
// perform a session swap before its own Run().
//
// Doesn't actually do the swap here — that has to happen at a clean
// boundary (no in-flight tensors, no held output Ort::Values), which
// is precisely the start of the next public method. Calling
// Reinitialize from inside the middle of an inference path while
// holding output_node_names / inputTensor / outputTensors references
// would invalidate them under our feet.
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
if (ms <= kSlowRecreateThresholdMs) return;
if (!_warmedUp) return; // first-shape compile is legitimately slow
// Cooldown gate — if we recreated recently, hold off. Avoids a
// recreate loop if the underlying issue is permanent (e.g. driver
// truly broken) rather than a transient hiccup.
const auto now = std::chrono::steady_clock::now();
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastRecreateAt).count();
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
sinceLast < kRecreateCooldownMs) {
ANS_DBG("ANSONNXRec_run",
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
context, ms, static_cast<long long>(sinceLast));
return;
}
_slowRecreatePending.store(true, std::memory_order_release);
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": slow Run (" << ms
<< " ms > threshold) — flagged for proactive session recreate\n";
ANS_DBG("ANSONNXRec_run",
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
context, ms, kSlowRecreateThresholdMs);
}
// AMD/DirectML latency-based recreate (Path B) — apply helper.
//
// Called at the entry of every public method (Recognize,
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
// armed by a previous call's RecordRunLatency, do the session swap
// here — before any of this call's tensors are constructed. Caller
// already holds _mutex, so the new ort_session is exclusively ours
// for the rest of this call.
//
// On Reinitialize failure we leave _slowRecreatePending=false so we
// don't loop forever; the existing exception-based HandleRunFailure-
// AndRecreate path will fall back to CPU on the next actual throw.
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
return;
}
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
<< " (latency-triggered)\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate on EP=%d (latency-triggered)",
context, static_cast<int>(currentEp));
try {
Reinitialize(currentEp);
_warmedUp = false;
_lastRecreateAt = std::chrono::steady_clock::now();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate succeeded; next call retries on fresh session\n";
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
} catch (const std::exception& e) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed: " << e.what()
<< " — keeping existing session, exception path will catch real failures\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate failed (%s)", context, e.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed (unknown)\n";
}
}
// AMD/DirectML self-heal helper. See header comment for detailed contract.
// In short: on Run() failure, recreate the session on the same EP first
// (transient driver hiccup recovery — the common case); on a second
// failure, fall back to CPU EP. Either path leaves ort_session in a
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
// result is already lost — caller returns empty.
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
const std::exception& e) {
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
<< ": " << e.what() << " — recreating session\n";
ANS_DBG("ANSONNXRec_run",
"%s: Run failed EP=%d msg=%.200s — recreating",
context, static_cast<int>(currentEp), e.what());
// Option A: a Run() exception on a DML session is the strongest
// signal the AMD driver can't sustain parallel inference. Trip
// process-wide DML serial mode so future inference calls (from
// any module) take the global mutex. Idempotent — only the
// first caller per process flips the flag. No-op on
// Intel/NVIDIA/CPU.
if (currentEp == EngineType::AMD_GPU) {
ANSCENTER::DmlSerialState::TripFromException(
context, e.what());
}
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
// are transient driver hiccups; a fresh session typically works. This
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
// throw from Run() under normal load, so this path is rarely taken.
if (currentEp != EngineType::CPU) {
try {
Reinitialize(currentEp);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": session recreated on EP=" << static_cast<int>(currentEp)
<< " — next call will retry on GPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: session recreated on EP=%d (next call retries on GPU)",
context, static_cast<int>(currentEp));
return;
} catch (const std::exception& reCreateErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed: " << reCreateErr.what()
<< " — falling back to CPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: same-EP recreate failed (%s); falling back to CPU",
context, reCreateErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
}
}
// Step 2: last resort — recreate on CPU. Slower but always works
// when the model itself is valid. Caller's next call uses CPU EP.
try {
Reinitialize(EngineType::CPU);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": fallback session created on CPU\n";
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
} catch (const std::exception& cpuErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed: " << cpuErr.what()
<< " — ort_session likely null, future calls will return empty\n";
ANS_DBG("ANSONNXRec_run",
"%s: CPU fallback failed (%s); ort_session may be null",
context, cpuErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed (unknown exception)\n";
}
}
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
std::lock_guard<std::mutex> lock(_mutex);
@@ -120,6 +279,12 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
return {};
}
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
// are built — so the rest of this method runs on a fresh session.
ApplyPendingSlowRecreate("Recognize");
if (!ort_session) return {}; // recreate could have failed; bail safely
try {
// Step 1: aspect-preserving resize to height=imgH_, width capped
// at imgMaxW_. Then round resized width up to the next bucket.
@@ -133,10 +298,24 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::Recognize");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
// Latency-based recreate (Path B): arms the flag for the NEXT call
// if this one was abnormally slow.
RecordRunLatency("Recognize", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -147,7 +326,11 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
return CTCDecode(outputData, seqLen, numClasses);
}
catch (const Ort::Exception& e) {
std::cerr << "[ONNXOCRRecognizer] Inference failed: " << e.what() << std::endl;
// AMD/DML self-heal: recreate session for next call (same EP first,
// CPU fallback as last resort). This call's output is lost — return
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
// tracker migrates plate text across frames.
HandleRunFailureAndRecreate("Recognize", e);
return {};
}
}
@@ -158,6 +341,13 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
std::vector<TextLine>& out) {
if (crops.empty()) return;
// Path B (AMD/DML): apply pending proactive recreate before any tensors
// are constructed for this batch. RecognizeBatch holds _mutex around
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
// same lock the caller already owns.
ApplyPendingSlowRecreate("RunBatchAtWidth");
if (!ort_session) return; // recreate may have failed; abort safely
try {
// Real number of crops vs. padded batch size submitted to ORT.
// Padding to the ladder keeps the recognizer's input-shape set
@@ -189,10 +379,22 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
*memory_info_handler, batchInput.data(), batchInput.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
RecordRunLatency("RunBatchAtWidth", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -215,11 +417,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
}
}
catch (const Ort::Exception& e) {
// ORT will throw if the model doesn't support a batch dimension > 1.
// Fall back to per-image inference for this group.
// The original recovery path here covered "model doesn't support
// batch dim >1" — a one-shot decision: drop to single-image and
// try each crop. We keep that, but ALSO add AMD/DML self-heal
// on top: if the per-image fallback ALSO throws, we recreate the
// session (same EP first, then CPU) so the next caller gets a
// healthy session. The current call's results stay empty for
// any crops we couldn't decode — caller tolerates this.
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
<< bucketW << " (" << e.what()
<< ") — falling back to single-image path." << std::endl;
bool anyPerImageFailed = false;
std::string lastPerImageErr;
for (size_t i = 0; i < crops.size(); ++i) {
// Direct call (we already hold _mutex via the public RecognizeBatch
// wrapper). Replicate the single-image preprocessing here to avoid
@@ -232,10 +441,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
std::vector<Ort::Value> outputTensors;
{
// Option A: per-image fallback path also goes
// through the DML serialization gate.
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType,
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
float* outData = outputTensors[0].GetTensorMutableData<float>();
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int seqLen = static_cast<int>(outShape[1]);
@@ -245,6 +462,20 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
<< e2.what() << std::endl;
out[origIndices[i]] = {};
anyPerImageFailed = true;
lastPerImageErr = e2.what();
}
}
// If even the per-image fallback path is throwing, the session is
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
// session (same EP first, CPU as last resort).
if (anyPerImageFailed) {
try {
std::runtime_error proxy(lastPerImageErr);
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
} catch (...) {
// HandleRunFailureAndRecreate already logs; swallow.
}
}
}
@@ -323,10 +554,13 @@ void ONNXOCRRecognizer::Warmup() {
// plates pay a one-time spike on first occurrence of batch=16/24.
auto totalT0 = std::chrono::high_resolution_clock::now();
int warmedShapes = 0;
int totalShapes = 0;
std::string lastWarmupErr;
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
const int batchN = kRecWarmupBatchLadder[bs];
for (int b = 0; b < kRecNumBuckets; ++b) {
const int bucketW = kRecBucketWidths[b];
++totalShapes;
try {
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
@@ -361,6 +595,7 @@ void ONNXOCRRecognizer::Warmup() {
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
<< batchN << " bucketW=" << bucketW << ": "
<< e.what() << std::endl;
lastWarmupErr = e.what();
}
}
}
@@ -368,6 +603,30 @@ void ONNXOCRRecognizer::Warmup() {
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
<< " shapes in " << totalMs << " ms total" << std::endl;
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
// production traffic will hit Run() failures on every call. Recreate
// it now (same EP first, CPU as fallback) so the pipeline can still
// serve. We deliberately keep _warmedUp = false in this path; the
// next RecognizeBatch will re-trigger Warmup on the recovered session.
//
// We do NOT trigger self-heal on partial failure — losing 2 of 16
// shapes is normal (e.g. some bucket dims aren't supported by all
// EPs); the per-shape catch already guards against that.
if (warmedShapes == 0 && totalShapes > 0) {
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
<< totalShapes << " shapes failed — triggering self-heal\n";
ANS_DBG("ANSONNXRec_run",
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
totalShapes, lastWarmupErr.c_str());
try {
std::runtime_error proxy(lastWarmupErr);
HandleRunFailureAndRecreate("Warmup", proxy);
} catch (...) {}
// Don't set _warmedUp = true; let the recovered session warm up
// itself on the next caller's invocation.
return;
}
_warmedUp = true;
}
@@ -5,6 +5,8 @@
#include <vector>
#include <string>
#include <mutex>
#include <atomic>
#include <chrono>
namespace ANSCENTER {
namespace onnxocr {
@@ -51,6 +53,62 @@ private:
// CTC greedy decode
TextLine CTCDecode(const float* outputData, int seqLen, int numClasses);
// AMD/DirectML self-heal helper. Called by Recognize / RunBatchAtWidth /
// Warmup when ort_session->Run() throws an Ort::Exception. Tries to
// recreate the session on the same EP first (transient driver hiccup
// recovery — the common case for DML 0x80004005 E_FAIL); if that fails,
// falls back to CPU EP. On full failure leaves ort_session in whatever
// state Reinitialize left it (possibly nullptr); the next Recognize
// call will short-circuit on the !ort_session check and return empty.
// Resets _warmedUp so the new session gets re-warmed on first use.
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
// ----------------------------------------------------------------
// AMD/DirectML latency-based self-heal (Path B).
//
// Observed AMD failure mode: after a transient driver event (e.g.
// helmet detector hits DML E_FAIL and CPU-falls-back), DML stays in
// a degraded state. OCR Run() calls do NOT throw, but their latency
// jumps from 50–200 ms to 1000–7000 ms, often rising over time. The
// exception-based self-heal above never fires because there's no
// exception. This proactive path observes the latency and forces a
// session recreate when a Run() exceeds a sanity threshold — the
// fresh session typically returns to baseline latency.
//
// How it works:
// 1. Each public method records its Run() duration.
// 2. If duration > kSlowRecreateThresholdMs AND we're past warmup
// AND we're past the cooldown window, set _slowRecreatePending.
// 3. The NEXT call to a public method checks the flag at entry; if
// set, it calls Reinitialize(currentEp) BEFORE its Run() — so
// the call that sees the flag uses a fresh session and returns
// to fast latency.
// 4. After a recreate, _lastRecreateMs starts the cooldown timer.
//
// Threshold tuning: 2 s catches the AMD-degraded individual Run()
// pattern observed in production (1–2 s per OCR Run() when DML is
// sick). Healthy individual Run() is 50–500 ms, so 2 s leaves
// comfortable headroom. First-shape compiles on OpenVINO can hit
// ~1.7 s but those happen during Warmup, which is gated out below
// via the _warmedUp check. 60 s cooldown prevents thrash if the
// driver is in a permanently bad state — at most one recreate per
// minute per session.
// ----------------------------------------------------------------
static constexpr double kSlowRecreateThresholdMs = 2000.0;
static constexpr int64_t kRecreateCooldownMs = 60000;
// Note: a slow Run() detected by RecordRunLatency arms this flag;
// ApplyPendingSlowRecreate (called at every public-method entry)
// performs the actual Reinitialize and clears the flag. std::atomic
// is used so the read at entry is lock-free; the write happens under
// the same _mutex the public methods already hold, so the only race
// we're guarding is "another public method picked up the flag first."
std::atomic<bool> _slowRecreatePending{false};
std::chrono::steady_clock::time_point _lastRecreateAt{};
void RecordRunLatency(const char* context, double ms);
void ApplyPendingSlowRecreate(const char* context);
std::vector<std::string> keys_;
int imgH_ = kRecImgH;
int imgMaxW_ = kRecImgMaxW;
+1 -1
View File
@@ -24,7 +24,7 @@ namespace ANSCENTER {
std::unique_ptr<IANSCustomClass> _customDetector;
HMODULE hMod= nullptr;
std::string FindANSCustomLibraryName(const std::string& folderPath, const std::string& keyword);
bool CreateCustomDetector();
bool CreateCustomDetector();
};
}
#endif
+51
View File
@@ -36,6 +36,57 @@ const int MAX_MISSING_SCREEN = 1000;
namespace ANSCENTER
{
// ────────────────────────────────────────────────────────────────────────
// DirectML fatal-error detection
//
// AMD's DirectML EP can fail in several distinct ways that all leave the
// Ort::Session permanently broken and require a fresh CPU-fallback session:
//
// 887A0005 DXGI_ERROR_DEVICE_REMOVED — adapter killed (RDP attach,
// TDR, sleep/wake, driver swap). GetDeviceRemovedReason gives
// the sub-cause but the device is gone either way.
// 80004005 E_FAIL — generic D3D12 / DML failure. Seen from
// DmlGraphFusionHelper::ExecuteOperator and
// DmlCommandRecorder::ExecuteCommandList on AMD when the
// driver couldn't compile/execute a fused subgraph or when
// TDR fires mid-command-list. No structured sub-cause
// available — must treat as device-fatal.
//
// Branding strings ("DmlFusedNode", "DmlGraphFusionHelper",
// "DmlCommandRecorder") catch future failure modes that surface the same
// way without us having to chase each new HRESULT.
//
// Used by every catch site that wraps Ort::Session::Run on AMD DML.
// Centralised here so a new failure mode is handled once, not 4+ times.
// ────────────────────────────────────────────────────────────────────────
inline bool IsDmlFatalError(const std::string& msg) {
return msg.find("887A0005") != std::string::npos
|| msg.find("80004005") != std::string::npos
|| msg.find("DmlFusedNode") != std::string::npos
|| msg.find("DmlGraphFusionHelper") != std::string::npos
|| msg.find("DmlCommandRecorder") != std::string::npos
|| msg.find("DmlExecutionProvider") != std::string::npos;
}
// ────────────────────────────────────────────────────────────────────────
// IsDmlDeviceRemoved — narrowly checks ONLY 887A0005
// (DXGI_ERROR_DEVICE_REMOVED). This is the truly unrecoverable case:
// the D3D12 adapter is gone (driver crashed, RDP attach killed the
// device, sleep/wake torched the WDDM context). No amount of session
// recreation will bring it back — only a process restart or CPU
// fallback can recover.
//
// In contrast, 80004005 (E_FAIL) from DmlGraphFusionHelper /
// DmlCommandRecorder is intermittent on AMD when the driver chokes on
// a fused subgraph or TDR fires mid-command-list. A fresh DML
// session typically clears it and resumes GPU inference — that's the
// stay-on-GPU recovery path used by ANSONNXYOLO's catch blocks.
// ────────────────────────────────────────────────────────────────────────
inline bool IsDmlDeviceRemoved(const std::string& msg) {
return msg.find("887A0005") != std::string::npos
|| msg.find("DXGI_ERROR_DEVICE_REMOVED") != std::string::npos;
}
template <typename T>
typename std::enable_if<std::is_arithmetic<T>::value, T>::type
inline clamp(const T& value, const T& low, const T& high)
+113 -197
View File
@@ -1,4 +1,5 @@
#include"ANSONNXCL.h"
#include "ANSONNXYOLO.h" // ANSONNXYOLO is the delegate target — see facade comment in ANSONNXCL.h
#include "EPLoader.h"
#include "OpenVINODeviceConfig.h"
#include <typeinfo> // typeid(e).name() in ANS_DBG diagnostics
@@ -876,9 +877,50 @@ namespace ANSCENTER
);
}
catch (const Ort::Exception& e) {
const std::string msg = e.what();
this->_logger.LogError("ANSONNXCL::classify",
"ONNX Runtime Exception during Run(): " + std::string(e.what()), __FILE__, __LINE__);
"ONNX Runtime Exception during Run(): " + msg, __FILE__, __LINE__);
delete[] blobPtr; // Clean up on error
blobPtr = nullptr;
// DML device-lost / DML graph-execution failure → tear down the
// dead DirectML session and rebuild on CPU. Without this, every
// subsequent classify() call hits the same dead session and
// floods the event log (16k+ entries observed on AMD/RDP).
if (IsDmlFatalError(msg) && !_dmlDeviceLost) {
_dmlDeviceLost = true;
_logger.LogFatal("ANSONNXCL::classify",
std::string("DirectML fatal error — recreating session on CPU. msg=")
+ msg.substr(0, 256),
__FILE__, __LINE__);
ANS_DBG("ANSONNXCL", "DML fatal — rebuild on CPU msg=%s", msg.c_str());
try {
// Order matters: drop AllocatedStringPtrs (which depend
// on the live session's allocator) BEFORE tearing down
// session_, sessionOptions_, env_. Init() will repopulate.
inputNodeNameAllocatedStrings_.clear();
outputNodeNameAllocatedStrings_.clear();
inputNames_.clear();
outputNames_.clear();
session_ = Ort::Session{ nullptr };
sessionOptions_ = Ort::SessionOptions{ nullptr };
env_ = Ort::Env{ nullptr };
if (Init(_modelFilePath, inputImageShape_, /*useGPU=*/false)) {
_logger.LogInfo("ANSONNXCL::classify",
"CPU fallback session created successfully",
__FILE__, __LINE__);
ANS_DBG("ANSONNXCL", "CPU fallback OK");
} else {
_logger.LogFatal("ANSONNXCL::classify",
"CPU fallback session creation failed",
__FILE__, __LINE__);
}
} catch (const std::exception& re) {
_logger.LogFatal("ANSONNXCL::classify",
std::string("CPU fallback exception: ") + re.what(),
__FILE__, __LINE__);
}
}
return {};
}
@@ -922,216 +964,90 @@ namespace ANSCENTER
return {};
}
}
bool ANSONNXCL::OptimizeModel(bool fp16, std::string & optimizedModelFolder) {
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
return true;
}
bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
ModelLoadingGuard mlg(_modelLoading);
try {
_modelLoadValid = false;
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::ONNXCL;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
_fp16 = (modelConfig.precisionType == PrecisionType::FP16);
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
// ────────────────────────────────────────────────────────────────────────
// Public-method facade: delegate everything to the wrapped ANSONNXYOLO.
//
// This is a verification harness — ANSONNXYOLO already covers the
// classification path (auto-detected from model output shape) and ships
// with the safe DML config + DML-fatal-error recovery. By routing every
// ANSONNXCL public call through ANSONNXYOLO we can A/B test whether
// ANSONNXCL's own session/postprocess logic is the actual cause of the
// AMD same-process DML races we saw with the custom helmet model.
//
// The dead code below the facade (Init / preprocess / postprocess /
// classify / Ort::* members in the header) is intentionally preserved
// so the original implementation is one git-revert away if needed.
// ────────────────────────────────────────────────────────────────────────
namespace {
// Lazy-create the delegate on first public call. Not thread-safe in
// the strict sense, but the public methods all take _mutex first via
// the underlying ANSODBase, so concurrent first-time creation is
// serialised in practice.
inline ANSONNXYOLO& GetDelegate(std::unique_ptr<ANSONNXYOLO>& slot) {
if (!slot) {
slot = std::make_unique<ANSONNXYOLO>();
ANS_DBG("ANSONNXCL_Facade",
"delegate created (ANSONNXYOLO=%p)", (void*)slot.get());
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// 2. Initialize ONNX Runtime session
cv::Size targetInputShape= cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
result = Init(_modelFilePath, targetInputShape,true);
_modelLoadValid = true;
_isInitialized = true;
return result;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSONNXCL::Initialize", e.what(), __FILE__, __LINE__);
return false;
return *slot;
}
}
bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
ModelLoadingGuard mlg(_modelLoading);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
_fp16 = true; // Load Model from Here
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Initialize ONNX Runtime session
cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
result = Init(_modelFilePath, targetInputShape,true);
_modelLoadValid = true;
_isInitialized = true;
return result;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSONNXCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
bool ANSONNXCL::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
ANS_DBG("ANSONNXCL_Facade", "OptimizeModel → ANSONNXYOLO");
return GetDelegate(_yoloDelegate).OptimizeModel(fp16, optimizedModelFolder);
}
bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
ModelLoadingGuard mlg(_modelLoading);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".onnx";
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
_fp16 = true; // Load Model from Here
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
_classFilePath = CreateFilePath(_modelFolder, className);
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// 2. Initialize ONNX Runtime session
cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
result = Init(_modelFilePath, targetInputShape,true);
_modelLoadValid = true;
_isInitialized = true;
return result;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSONNXCL::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
return false;
}
bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath,
const std::string& modelZipPassword,
std::string& labelMap) {
ANS_DBG("ANSONNXCL_Facade", "Initialize → ANSONNXYOLO zip=%s",
modelZipFilePath.c_str());
return GetDelegate(_yoloDelegate).Initialize(
licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
}
bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword) {
ANS_DBG("ANSONNXCL_Facade", "LoadModel → ANSONNXYOLO zip=%s",
modelZipFilePath.c_str());
return GetDelegate(_yoloDelegate).LoadModel(modelZipFilePath, modelZipPassword);
}
bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap) {
ANS_DBG("ANSONNXCL_Facade",
"LoadModelFromFolder → ANSONNXYOLO modelName=%s folder=%s",
modelName.c_str(), modelFolder.c_str());
return GetDelegate(_yoloDelegate).LoadModelFromFolder(
licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
}
bool ANSONNXCL::Destroy() {
return true;
if (_yoloDelegate) {
ANS_DBG("ANSONNXCL_Facade", "Destroy → ANSONNXYOLO");
return _yoloDelegate->Destroy();
}
return true;
}
ANSONNXCL::~ANSONNXCL() {
// Explicit Destroy() so the delegate's session is torn down before
// the unique_ptr destructor frees the object. unique_ptr<ANSONNXYOLO>
// still releases the heap memory afterwards.
Destroy();
}
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& input, const std::string& camera_id) {
if (!PreInferenceCheck("ANSONNXCL::RunInference")) return {};
try {
std::vector<Object> result;
if (input.empty()) return result;
if ((input.cols < 5) || (input.rows < 5)) return result;
return classify(input, camera_id);
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSONNXCL::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& input,
const std::string& camera_id) {
if (input.empty()) return {};
if (input.cols < 5 || input.rows < 5) return {};
return GetDelegate(_yoloDelegate).RunInference(input, camera_id);
}
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& inputImgBGR) {
return RunInference(inputImgBGR, "CustomCam");
}
+31
View File
@@ -3,8 +3,26 @@
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include <memory>
namespace ANSCENTER {
// Forward declaration: avoid pulling ANSONNXYOLO.h into every consumer of
// ANSONNXCL.h. Definition is included in ANSONNXCL.cpp.
class ANSONNXYOLO;
// ────────────────────────────────────────────────────────────────────────
// ANSONNXCL — verification facade over ANSONNXYOLO.
//
// ANSONNXYOLO already auto-detects classification models (single 2D
// output [B, nc]) and routes through its own classification postprocess.
// It also has the safe DML config (DisableMemPattern + ORT_SEQUENTIAL +
// EXTENDED) and the IsDmlFatalError catch-and-rebuild path.
//
// We delegate every public method here to an internal ANSONNXYOLO so we
// can A/B test whether ANSONNXCL's own session/postprocess code is the
// real cause of the AMD same-process DML races we saw with the custom
// helmet model. Public ABI is unchanged — external callers don't move.
// ────────────────────────────────────────────────────────────────────────
class ANSENGINE_API ANSONNXCL :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
@@ -15,10 +33,23 @@ namespace ANSCENTER {
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSONNXCL();
private:
// The wrapped engine. Lazy-initialised on first public call so we
// don't pay the cost (and pull in the dependency) for callers who
// never use this class.
std::unique_ptr<ANSONNXYOLO> _yoloDelegate;
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
// DML device-lost recovery: when DirectML fails fatally
// (HRESULT 887A0005 device-removed, 80004005 graph/command-recorder
// E_FAIL, etc. — see ANSCENTER::IsDmlFatalError) the Ort::Session
// is permanently broken. We detect once, recreate on CPU via
// Init(_modelFilePath, inputImageShape_, /*useGPU=*/false), and
// suppress further error-log flooding via this flag.
bool _dmlDeviceLost{ false };
size_t vectorProduct(const std::vector<int64_t>& vector);
bool preprocessImageToTensor(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& targetShape,
+266 -92
View File
@@ -1610,7 +1610,7 @@ namespace ANSCENTER {
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
return false;
}
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath);
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath);
return true;
}
catch (const std::exception& e) {
@@ -1626,7 +1626,7 @@ namespace ANSCENTER {
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
return false;
}
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath, engineType);
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath, engineType);
return true;
}
catch (const std::exception& e) {
@@ -1638,26 +1638,35 @@ namespace ANSCENTER {
// ========================================================================
// WarmUpEngine — run a dummy inference after session creation.
//
// Scope: **NVIDIA (CUDA EP) only.** On first inference, the CUDA EP
// allocates its memory arena (capped at 2 GB via BasicOrtHandler config),
// resolves cuDNN convolution algorithms, and populates the kernel launch
// cache. Running one dummy inference at load time amortises this cost
// so the first real frame doesn't see a latency spike.
// Scope: NVIDIA (CUDA EP) AND AMD (DirectML EP).
// • NVIDIA: amortises CUDA arena allocation, cuDNN convolution algo
// selection, and kernel-launch cache population.
// • AMD: amortises D3D12 PSO compilation and first-time DML kernel
// binding. Empirically ~150 ms on DirectML (see comment in
// ONNXOCRRecognizer::Warmup). Without this, that compile cost
// surfaces inside the first live Run() — which on AMD has been
// observed to race with concurrent DML traffic from other sessions
// (e.g. helmet detector running while ALPR's _lpDetector fires its
// first ever Run()) and trigger DmlGraphFusionHelper crashes /
// 887A0006 DEVICE_HUNG. Moving the heavy first-run work into
// LoadEngine — where only the loading thread is active — sidesteps
// this timing window.
//
// Explicitly disabled on AMD, Intel and CPU:
// • AMD (DirectML) — calling detect() at load time has been observed
// to hit a multi-terabyte cv::Mat allocation inside postprocessLegacy
// on AMD RDNA iGPUs when DirectML returns garbage output tensor
// dims. ONNXYOLO::detect() now has an output-shape sanity guard
// that catches this at runtime, so the warm-up would add risk
// without benefit. Earlier builds enabled warm-up specifically for
// Radeon 680M TDR mitigation; that workaround is obsolete with
// current DirectML 1.15.x drivers.
// • Intel (OpenVINO) — running detect() at load time has been
// observed to expose latent heap-corruption bugs
// (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
// • CPU EP — no shader compile or kernel cache to warm up; the first
// real frame has the same latency as any subsequent frame.
// Skipped for:
// • Intel (OpenVINO EP) — OpenVINO compiles the full plan at session
// creation, so first Run() has no extra compile cost. Calling
// detect() at load time has also exposed latent heap-corruption
// bugs (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
// • CPU EP — no shader / kernel cache to warm up.
//
// History note: AMD warm-up was disabled in earlier builds after
// observing multi-terabyte cv::Mat allocations from garbage DirectML
// output dims. ONNXYOLO::detect() now has an output-shape sanity
// guard (see line ~257, kMaxOutputDim = 1,000,000) that catches that
// case before postprocessLegacy can allocate, so re-enabling AMD
// warm-up is safe. The exception-swallowing catch blocks below
// guarantee that any warm-up failure is non-fatal — the session is
// already loaded and real inference will still work.
//
// Non-fatal on failure: if warm-up itself throws, regular inference
// still works — the engine is fully loaded before WarmUpEngine runs.
@@ -1665,9 +1674,11 @@ namespace ANSCENTER {
void ANSONNXYOLO::WarmUpEngine() {
if (!m_ortEngine) return;
// Gate strictly on NVIDIA_GPU. Every other EP is a no-op.
if (m_ortEngine->getEngineType() != EngineType::NVIDIA_GPU) {
ANS_DBG("ONNXYOLO", "Warm-up skipped (non-NVIDIA EP)");
// Warm up on NVIDIA and AMD (both have lazy first-Run() compile cost).
// Skip on CPU and OpenVINO — see scope comment above.
const auto et = m_ortEngine->getEngineType();
if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) {
ANS_DBG("ONNXYOLO", "Warm-up skipped (CPU/OpenVINO EP)");
return;
}
@@ -1696,7 +1707,8 @@ namespace ANSCENTER {
// preprocessing (114,114,114 ~ 128) and avoids degenerate inputs.
cv::Mat dummy(h, w, CV_8UC3, cv::Scalar(128, 128, 128));
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy CUDA inference (%dx%d)", w, h);
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy %s inference (%dx%d)",
(et == EngineType::NVIDIA_GPU) ? "CUDA" : "DirectML", w, h);
auto t0 = std::chrono::steady_clock::now();
(void)m_ortEngine->detect(dummy, _classes,
@@ -1975,6 +1987,150 @@ namespace ANSCENTER {
}
}
// ====================================================================
// HandleDmlRunFailure — stay-on-DML recovery (replaces CPU-fallback).
//
// Called from every catch block that wraps an Ort::Session::Run on a
// DirectML EP. The contract is documented on the member declaration
// in ANSONNXYOLO.h — quick summary:
//
// * CAS on _dmlRecreateInProgress elects ONE recreate owner per
// driver hiccup; other concurrent failures just bail.
// * Cooldown (kDmlRecreateCooldownMs) prevents thrashing if E_FAIL
// keeps firing back to back.
// * IsDmlDeviceRemoved (887A0005) → CPU fallback, last resort.
// * After kMaxConsecutiveDmlFailures recreates without an
// intervening successful inference → CPU fallback.
// * Otherwise: reset() + InitOrtEngine(<original DML EP>) +
// WarmUpEngine() → fresh DML session, GPU inference resumes.
//
// Successful inferences reset _dmlConsecutiveFailures to 0 in
// DetectObjects / DetectObjectsBatch, so isolated transient hiccups
// do not accumulate toward the CPU-fallback threshold.
// ====================================================================
void ANSONNXYOLO::HandleDmlRunFailure(const char* context, const std::string& msg)
{
// Option A: a DML exception is the strongest possible signal
// that the AMD driver can't sustain parallel inference. Trip
// process-wide DML serial mode immediately so the NEXT
// inference attempt (from any thread, in this DLL or another)
// takes the global mutex and runs one-at-a-time. This is
// idempotent — only the first caller per process actually
// flips the flag. Done before the CAS so even the CAS losers
// (concurrent threads catching the same fatal) contribute the
// signal — but in practice the first one wins.
ANSCENTER::DmlSerialState::TripFromException(context, msg.c_str());
// CAS — only one thread owns the recreate at a time. Other
// concurrent DML failures just return and retry on the next call.
bool expected = false;
if (!_dmlRecreateInProgress.compare_exchange_strong(expected, true)) {
ANS_DBG("ONNXYOLO", "%s: DML fatal — another thread is recreating, skip",
context);
return;
}
// Hold the engine mutex for the entire recreate so concurrent
// inference threads (which use a shared_ptr local copy of the
// engine) don't see a half-built session.
std::lock_guard<std::recursive_mutex> lk(_mutex);
try {
// Cooldown — if we just recreated within the window, the
// driver is in a prolonged bad state; don't burn time
// recreating again immediately. Other threads will keep
// returning {} until either the next call past cooldown
// succeeds or we hit the CPU-fallback threshold.
const auto now = std::chrono::steady_clock::now();
const auto sinceLastMs = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastDmlRecreateAt).count();
if (_lastDmlRecreateAt.time_since_epoch().count() != 0
&& sinceLastMs < kDmlRecreateCooldownMs) {
ANS_DBG("ONNXYOLO",
"%s: DML fatal — within cooldown (%lld ms < %lld), skip",
context, (long long)sinceLastMs, (long long)kDmlRecreateCooldownMs);
_dmlRecreateInProgress.store(false);
return;
}
_lastDmlRecreateAt = now;
// Decide target EP for the new session:
// * 887A0005 device-removed → CPU is the only option
// * Too many consecutive failures → CPU as last resort
// * Otherwise → recreate on the SAME DML EP we were on
ANSCENTER::EngineType originalEp = ANSCENTER::EngineType::CPU;
if (m_ortEngine) {
originalEp = m_ortEngine->getEngineType();
}
const int failures = _dmlConsecutiveFailures.fetch_add(1) + 1;
const bool deviceRemoved = ANSCENTER::IsDmlDeviceRemoved(msg);
const bool exceededRetries = failures > kMaxConsecutiveDmlFailures;
ANSCENTER::EngineType targetEp = originalEp;
const char* reason = "stay-on-DML retry";
if (deviceRemoved) {
targetEp = ANSCENTER::EngineType::CPU;
reason = "device-removed (887A0005) → CPU fallback";
} else if (exceededRetries) {
targetEp = ANSCENTER::EngineType::CPU;
reason = "exceeded retry budget → CPU fallback";
}
_logger.LogFatal(context,
std::string("DirectML fatal — ") + reason
+ " (failure #" + std::to_string(failures)
+ "). msg=" + msg.substr(0, 256),
__FILE__, __LINE__);
ANS_DBG("ONNXYOLO",
"%s: DML fatal recreate — %s, failures=%d, originalEp=%d, targetEp=%d",
context, reason, failures, (int)originalEp, (int)targetEp);
// reset() releases the member's reference; concurrent inference
// threads holding a shared_ptr local copy keep their old
// engine alive until they return — no use-after-free (B2).
m_ortEngine.reset();
if (!InitOrtEngine(targetEp)) {
_logger.LogFatal(context,
"DML recreate failed — m_ortEngine left null", __FILE__, __LINE__);
ANS_DBG("ONNXYOLO", "%s: InitOrtEngine(%d) failed", context, (int)targetEp);
_dmlRecreateInProgress.store(false);
return;
}
// Warm up the new session so the first real call doesn't pay
// the algorithm-selection / shader-compile tax. AMD DML
// recreate WITHOUT warmup typically hits 1-2 s on the very
// first inference, which can re-trigger our 2 s slow watchdog
// and produce a false alarm.
try {
WarmUpEngine();
} catch (const std::exception& we) {
ANS_DBG("ONNXYOLO", "%s: WarmUpEngine after recreate threw: %s",
context, we.what());
}
// If we landed on CPU as a last resort, leave the failure
// counter pinned so we don't oscillate back to DML on the
// next exception. Otherwise (we're back on DML) DON'T reset
// the counter here — it gets reset by a *successful*
// inference in DetectObjects / DetectObjectsBatch.
if (targetEp == ANSCENTER::EngineType::CPU
&& originalEp != ANSCENTER::EngineType::CPU) {
_logger.LogInfo(context,
"Permanently switched to CPU EP — DML retry budget exhausted "
"or device removed. Restart the process to retry GPU.",
__FILE__, __LINE__);
}
}
catch (const std::exception& re) {
_logger.LogFatal(context,
std::string("DML recreate exception: ") + re.what(),
__FILE__, __LINE__);
}
_dmlRecreateInProgress.store(false);
}
std::vector<Object> ANSONNXYOLO::DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id)
{
@@ -1985,16 +2141,27 @@ namespace ANSCENTER {
return {};
}
// ── Snapshot config under a brief lock ──────────────────
// ── Snapshot config + engine handle under a brief lock ──
// Only hold _mutex long enough to validate state and copy
// the parameters needed for inference. The actual ORT
// detect() call runs outside the lock so that concurrent
// Initialize/LoadModel calls are not blocked for the full
// duration of inference.
//
// B2: We also snapshot a shared_ptr COPY of m_ortEngine
// here. The lock-free detect() call below dereferences
// the LOCAL copy (`engine`), not the member. If a
// concurrent thread catches a DML fatal and does
// m_ortEngine.reset() under the same lock, that only
// releases the member's reference — our local copy keeps
// the engine alive until this function returns. This
// closes the use-after-free race that produced the
// 0xc0000005 crash at ANSODEngine.dll +0x111fd0.
float probThresh, nmsThresh;
int numKps;
std::vector<std::string> classes;
bool trackerEnabled, stabilizationEnabled;
std::shared_ptr<ONNXYOLO> engine;
{
ANS_DBG("ONNXYOLO", "DetectObjects: cam=%s acquiring mutex...", camera_id.c_str());
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjects");
@@ -2007,13 +2174,14 @@ namespace ANSCENTER {
return {};
}
// Snapshot parameters while locked
// Snapshot parameters + engine handle while locked
probThresh = PROBABILITY_THRESHOLD;
nmsThresh = NMS_THRESHOLD;
numKps = NUM_KPS;
classes = _classes;
trackerEnabled = _trackerEnabled;
stabilizationEnabled = _stabilizationEnabled;
engine = m_ortEngine; // ref-count++
}
// ── _mutex released — heavy work below runs lock-free ───
@@ -2047,11 +2215,28 @@ namespace ANSCENTER {
}
}
// Run ORT inference — no mutex held, this is the expensive call
auto results = m_ortEngine->detect(inferenceImage, classes,
probThresh,
nmsThresh,
numKps);
// Run ORT inference — no engine mutex held, this is the
// expensive call.
// B2: dereference the LOCAL shared_ptr `engine`, not the
// member m_ortEngine. A concurrent recreate thread may
// reset() m_ortEngine while we're inside detect() — the
// local ref-count keeps the underlying ONNXYOLO alive
// until this function returns.
//
// Option A: DmlInferenceGuard wraps the call to (a)
// serialize on the global DML mutex IF DML serial mode has
// been tripped, and (b) trip serial mode if THIS call's
// latency exceeds the slow threshold (1.5 s — well below
// WDDM TDR). No-op on Intel/NVIDIA/CPU.
std::vector<Object> results;
{
DmlInferenceGuard dmlGuard(engine->getEngineType(),
"ANSONNXYOLO::DetectObjects");
results = engine->detect(inferenceImage, classes,
probThresh,
nmsThresh,
numKps);
}
// --- Rescale coordinates from full-res back to display-res ---
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
@@ -2075,42 +2260,38 @@ namespace ANSCENTER {
for (auto& obj : results)
obj.cameraId = camera_id;
// Tracking/stabilization (ApplyTracking has its own lock)
if (trackerEnabled && !m_ortEngine->lastWasClassification) {
// Tracking/stabilization (ApplyTracking has its own lock).
// B2: read lastWasClassification from the LOCAL engine copy
// — using m_ortEngine here would re-introduce the race we
// just closed for detect().
if (trackerEnabled && !engine->lastWasClassification) {
results = ApplyTracking(results, camera_id);
if (stabilizationEnabled) results = StabilizeDetections(results, camera_id);
}
// A successful inference clears the DML failure budget so
// isolated transient hiccups (one E_FAIL every few minutes)
// never accumulate toward kMaxConsecutiveDmlFailures and
// wrongly demote us to CPU. Only sustained back-to-back
// failures with no successful inference between them
// exhaust the budget.
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
return results;
}
catch (const std::exception& e) {
const std::string msg = e.what();
if (msg.find("887A0005") != std::string::npos) {
if (!_dmlDeviceLost) {
_dmlDeviceLost = true;
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
__FILE__, __LINE__);
ANS_DBG("ONNXYOLO", "DML device lost — recreating session on CPU");
try {
std::lock_guard<std::recursive_mutex> lk(_mutex);
m_ortEngine.reset();
if (InitOrtEngine(ANSCENTER::EngineType::CPU)) {
_logger.LogInfo("ANSONNXYOLO::DetectObjects",
"CPU fallback session created successfully",
__FILE__, __LINE__);
ANS_DBG("ONNXYOLO", "CPU fallback OK");
} else {
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
"CPU fallback session creation failed",
__FILE__, __LINE__);
}
} catch (const std::exception& re) {
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
std::string("CPU fallback exception: ") + re.what(),
__FILE__, __LINE__);
}
}
// Any DML-fatal HRESULT or DML branded error → rebuild on CPU.
// See IsDmlFatalError() in ANSEngineCommon.h for the full set
// (887A0005 device-removed, 80004005 generic E_FAIL from
// DmlGraphFusionHelper / DmlCommandRecorder, etc.).
if (IsDmlFatalError(msg)) {
// Stay-on-DML retry — see HandleDmlRunFailure for full
// contract. Tries to rebuild the session on the same
// DML EP first; only falls back to CPU on
// device-removed (887A0005) or after the retry budget
// is exhausted.
HandleDmlRunFailure("ANSONNXYOLO::DetectObjects", msg);
return {};
}
@@ -2157,19 +2338,9 @@ namespace ANSCENTER {
}
catch (const std::exception& e) {
const std::string msg = e.what();
if (msg.find("887A0005") != std::string::npos) {
if (!_dmlDeviceLost) {
_dmlDeviceLost = true;
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
__FILE__, __LINE__);
try {
m_ortEngine.reset();
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
"CPU fallback session creation failed", __FILE__, __LINE__);
} catch (...) {}
}
if (IsDmlFatalError(msg)) {
// Stay-on-DML retry — see HandleDmlRunFailure.
HandleDmlRunFailure("ANSONNXYOLO::RunInferencesBatch", msg);
return {};
}
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
@@ -2184,11 +2355,15 @@ namespace ANSCENTER {
try {
if (_modelLoading.load()) return {};
// Snapshot config under brief lock
// Snapshot config + engine handle under brief lock.
// B2: shared_ptr copy `engine` keeps the underlying ONNXYOLO
// alive across the lock-free detectBatch() call even if a
// concurrent DML-fatal handler resets m_ortEngine.
float probThresh, nmsThresh;
int numKps;
std::vector<std::string> classes;
bool trackerEnabled, stabilizationEnabled;
std::shared_ptr<ONNXYOLO> engine;
{
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjectsBatch");
if (!lk.owns_lock()) return {};
@@ -2204,13 +2379,20 @@ namespace ANSCENTER {
classes = _classes;
trackerEnabled = _trackerEnabled;
stabilizationEnabled = _stabilizationEnabled;
engine = m_ortEngine; // ref-count++
}
// Heavy work outside lock
auto batchResults = m_ortEngine->detectBatch(
inputImages, classes, probThresh, nmsThresh, numKps);
// Heavy work outside lock — dereference local copy, not member.
// Option A: same DmlInferenceGuard contract as DetectObjects.
std::vector<std::vector<Object>> batchResults;
{
DmlInferenceGuard dmlGuard(engine->getEngineType(),
"ANSONNXYOLO::DetectObjectsBatch");
batchResults = engine->detectBatch(
inputImages, classes, probThresh, nmsThresh, numKps);
}
const bool isClassification = m_ortEngine->lastBatchWasClassification;
const bool isClassification = engine->lastBatchWasClassification;
for (auto& results : batchResults) {
for (auto& obj : results)
@@ -2222,24 +2404,16 @@ namespace ANSCENTER {
}
}
// Clear DML failure budget on success — see DetectObjects
// for the full rationale.
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
return batchResults;
}
catch (const std::exception& e) {
const std::string msg = e.what();
if (msg.find("887A0005") != std::string::npos) {
if (!_dmlDeviceLost) {
_dmlDeviceLost = true;
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
__FILE__, __LINE__);
try {
std::lock_guard<std::recursive_mutex> lk(_mutex);
m_ortEngine.reset();
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
"CPU fallback session creation failed", __FILE__, __LINE__);
} catch (...) {}
}
if (IsDmlFatalError(msg)) {
// Stay-on-DML retry — see HandleDmlRunFailure.
HandleDmlRunFailure("ANSONNXYOLO::DetectObjectsBatch", msg);
return {};
}
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
+67 -7
View File
@@ -210,14 +210,74 @@ namespace ANSCENTER {
int NUM_KPS{ 0 };
float KPS_THRESHOLD{ 0.5f };
// ONNX Runtime inference engine
std::unique_ptr<ONNXYOLO> m_ortEngine;
// ONNX Runtime inference engine.
//
// B2: shared_ptr (was unique_ptr) so concurrent inference threads
// can grab a ref-counted local copy under the brief snapshot lock
// and dereference it OUTSIDE the lock without racing with a
// concurrent reset() in the DML-fatal CPU-fallback path. Without
// this, a thread doing `m_ortEngine->detect(...)` lock-free could
// see its underlying object freed mid-call by another thread that
// caught a DML E_FAIL and rebuilt the session — that was the
// 0xc0000005 access violation observed in ANSODEngine.dll under
// AMD/DirectML when multiple AI tasks ran in parallel.
std::shared_ptr<ONNXYOLO> m_ortEngine;
// DML device-lost recovery: when DirectML's GPU device is removed
// (HRESULT 887A0005), the session is permanently broken. We detect
// this once, attempt a CPU-fallback recreation, and suppress further
// error-log flooding.
bool _dmlDeviceLost{ false };
// ────────────────────────────────────────────────────────────────
// DML stay-on-GPU recovery state.
//
// AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from
// DmlGraphFusionHelper / DmlCommandRecorder when the driver
// chokes under concurrent inference pressure. Empirically this
// is NOT a permanent device loss — recreating the ORT session on
// the same DML EP clears the wedged state and resumes GPU
// inference at full speed. Only the truly-fatal 887A0005
// (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback.
//
// Recovery contract (used by every Run() catch block):
// 1. compare_exchange on _dmlRecreateInProgress elects one
// thread as the recreate owner. Other threads return {}
// and retry on the next call.
// 2. The owner checks the cooldown — if we recreated in the
// last kDmlRecreateCooldownMs ms, skip (driver is in a
// prolonged bad state, don't thrash).
// 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit
// kMaxConsecutiveDmlFailures recreates without a successful
// inference in between, fall back to CPU as last resort.
// 4. Otherwise reset() + InitOrtEngine(<original DML EP>) +
// WarmUpEngine() to rebuild on the same GPU EP.
// 5. On successful inference, _dmlConsecutiveFailures resets
// to 0, so isolated transient hiccups don't accumulate
// toward the CPU-fallback threshold.
//
// This replaces the old single-shot `bool _dmlDeviceLost` flag
// which latched on the first failure and forced CPU forever —
// user requirement is to keep AI tasks running on GPU, not to
// trade GPU performance for stability the moment the driver
// hiccups once.
// ────────────────────────────────────────────────────────────────
std::atomic<bool> _dmlRecreateInProgress{ false };
std::atomic<int> _dmlConsecutiveFailures{ 0 };
std::chrono::steady_clock::time_point _lastDmlRecreateAt{};
// Tunables — tuned for AMD Radeon 780M observed behavior.
// threshold: catches isolated transient AMD DML hiccups. After
// 3 consecutive recreates without a successful inference, the
// driver is in a prolonged bad state and CPU is the only
// forward path.
// cooldown: prevents recreate thrash. AMD DML session creation
// takes 200–500 ms; if E_FAIL fires from many threads at once
// we don't want to recreate N times in a row.
static constexpr int kMaxConsecutiveDmlFailures = 3;
static constexpr int64_t kDmlRecreateCooldownMs = 5000;
// Centralised DML-fatal recovery used by all 3 catch sites.
// Caller must have already determined the exception is DML-related
// (IsDmlFatalError == true). msg is the exception message used
// both to log and to detect the device-removed sub-case. Returns
// immediately without doing anything when another thread already
// owns the recreate (CAS loser path).
void HandleDmlRunFailure(const char* context, const std::string& msg);
// Internal detection pipeline
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
+7 -3
View File
@@ -384,13 +384,17 @@ namespace ANSCENTER {
catch (const std::exception& e) {
const std::string msg = e.what();
// DML device-removal detection (see ANSONNXYOLO.cpp for details)
if (msg.find("887A0005") != std::string::npos) {
// DML fatal-error detection (887A0005 device-removed, 80004005 generic
// DML E_FAIL, branded DmlFusedNode / DmlCommandRecorder / etc.).
// See ANSCENTER::IsDmlFatalError in ANSEngineCommon.h.
if (IsDmlFatalError(msg)) {
if (!_dmlDeviceLost) {
_dmlDeviceLost = true;
_logger.LogFatal("ANSOVFD::RunInference",
"DirectML GPU device lost (887A0005) — will attempt CPU fallback on next call",
std::string("DirectML fatal error — will attempt CPU fallback on next call. msg=")
+ msg.substr(0, 256),
__FILE__, __LINE__);
ANS_DBG("ANSOVFD", "DML fatal msg=%s", msg.c_str());
}
return {};
}
File diff suppressed because it is too large Load Diff