Fix AMD ALPR_OCR and custom model
This commit is contained in:
@@ -149,7 +149,36 @@
|
||||
"Bash(awk -F'[:\\(\\)]' '{gsub\\(/^ +| +$/,\"\",$2\\); split\\($2,a,\" \"\\); ms=a[1]; if \\(ms+0 >= 500\\) print $0}')",
|
||||
"Bash(awk -F'[:\\(\\)]' '{split\\($2,a,\" \"\\); ms=a[1]; if \\(ms+0 >= 100\\) print $0}')",
|
||||
"Bash(grep -cE \"^\\\\s*Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log4.txt\")",
|
||||
"Bash(grep -oE \"Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log5.txt\")"
|
||||
"Bash(grep -oE \"Plate: [^ ]+$\" \"C:/Users/nghia/Downloads/Log5.txt\")",
|
||||
"Bash(awk -F'\\\\t' '{print $3}')",
|
||||
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS_twoconsoleapps.log\")",
|
||||
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS_issue.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0 >= 49 && $2+0 <= 52 {print NR\": \"$0}' \"C:/Users/nghia/Downloads/AVNET-8845HS_issue.log\")",
|
||||
"Bash(awk -F'\\\\t' 'BEGIN{prev=0} {t=$2+0; if\\(t-prev>5\\){print NR\": GAP=\"t-prev\"s \"$0} prev=t}' \"C:/Users/nghia/Downloads/AVNET-8845HS3.log\")",
|
||||
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS3.log\")",
|
||||
"Bash(awk -F'\\\\t' '{print $3}' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=20 && $2+0<=22' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=20.5 && $2+0<=21.5' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=21.0 && $2+0<=21.7' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=21.05 && $2+0<=21.23' \"C:/Users/nghia/Downloads/AVNET-8845HS4_5.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=17 && $2+0<=20' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=17.4 && $2+0<=17.7' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")",
|
||||
"Bash(awk -F'\\\\t' '$2+0>=12 && $2+0<=18' \"C:/Users/nghia/Downloads/AVNET-8845HS6.log\")",
|
||||
"Bash(git -C \"C:/Projects/CLionProjects/ANSCORE\" status)",
|
||||
"Bash(git -C \"C:/Projects/CLionProjects/ANSCORE\" diff --stat modules/ANSODEngine/ANSONNXCL.h modules/ANSODEngine/ANSONNXCL.cpp modules/ANSODEngine/ANSONNXYOLO.cpp)",
|
||||
"Bash(awk -F'\\\\t' '{print $4}' \"/c/Users/nghia/Downloads/AVNET-8845HS7.log\")",
|
||||
"Bash(\"C:/Users/nghia/AppData/Local/Programs/CLion 2026.1/bin/cmake/win/x64/bin/cmake.exe\" --build C:/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSLPR-UnitTest -j 30)",
|
||||
"Bash(cmd.exe /c '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build C:/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSLPR-UnitTest -j 30')",
|
||||
"Bash(cmd.exe //c 'call \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\" --target ANSLPR-UnitTest -j 30')",
|
||||
"PowerShell($env:VS_VARS = '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\"'; $cmd = \"call $env:VS_VARS && `\"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe`\" --build `\"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release`\" --target ANSLPR-UnitTest -j 30\"; cmd.exe /c $cmd 2>&1 | Select-Object -Last 60)",
|
||||
"PowerShell(cmd.exe /c $cmd 2>&1)",
|
||||
"PowerShell($cmd = 'call \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvars64.bat\" >NUL && \"C:\\\\Users\\\\nghia\\\\AppData\\\\Local\\\\Programs\\\\CLion 2026.1\\\\bin\\\\cmake\\\\win\\\\x64\\\\bin\\\\cmake.exe\" --build \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\" --target ANSLPR-UnitTest -j 30')",
|
||||
"Bash(command -v dumpbin)",
|
||||
"Bash(dumpbin /SYMBOLS \"/c/Projects/CLionProjects/ANSCORE/cmake-build-release/bin/ANSOCR.dll\")",
|
||||
"Bash(\"/c/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/dumpbin.exe\" //SYMBOLS \"/c/Projects/CLionProjects/ANSCORE/cmake-build-release/bin/ANSOCR.dll\")",
|
||||
"Bash(cmake --build . --target ANSODEngine -j 8)",
|
||||
"PowerShell(@\\(Get-ChildItem -Path \"C:\\\\Program Files\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) + @\\(Get-ChildItem -Path \"C:\\\\Program Files \\(x86\\)\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) | ForEach-Object { $_.FullName })",
|
||||
"PowerShell($d = Get-Item C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\\\\bin\\\\ANSODEngine.dll; \"$\\($d.Name\\) | $\\($d.Length\\) bytes | $\\($d.LastWriteTime\\)\")"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
#pragma once
|
||||
// av_frame_ptr.h — RAII smart pointer for FFmpeg AVFrame*.
|
||||
//
|
||||
// Why this exists
|
||||
// ---------------
|
||||
// CVideoPlayer::getNV12Frame() / getCudaHWFrame() return cloned AVFrame* and
|
||||
// the contract is "caller must av_frame_free()". That contract is comment-only
|
||||
// — nothing in the type system enforces it, so any early return, exception,
|
||||
// or refactor that drops the pointer leaks the AVFrame *and* the underlying
|
||||
// NV12/YUV420P/YUVJ420P buffers (~3 MB per 1080p frame, ~50 MB/s under load).
|
||||
//
|
||||
// AvFramePtr wraps the raw pointer in std::unique_ptr<AVFrame, AvFrameDeleter>
|
||||
// so the destructor frees it automatically — leak-by-omission becomes a
|
||||
// compile-time impossibility on every internal code path that adopts this type.
|
||||
//
|
||||
// Scope: INTERNAL TO ANSCORE ONLY.
|
||||
// -------------------------------
|
||||
// Do NOT change DLL-exported function signatures to use AvFramePtr.
|
||||
// std::unique_ptr cannot cross a DLL boundary safely (different runtimes,
|
||||
// different STL implementations, different allocators). The public
|
||||
// AVFrame*-returning APIs in ANSRTSP.h / ANSSRT.h / etc. must remain raw
|
||||
// pointers for ABI stability.
|
||||
//
|
||||
// Use this type at the internal *consumer* call site:
|
||||
//
|
||||
// AvFramePtr nv12{ player->getNV12Frame() };
|
||||
// if (nv12) {
|
||||
// doSomething(nv12.get()); // borrowed access
|
||||
// } // auto-freed on scope exit
|
||||
//
|
||||
// To transfer ownership into ANSCV's pendingFree pipeline (the
|
||||
// gpu_frame_attach / gpu_frame_attach_cuda functions, which take raw
|
||||
// AVFrame* and store/defer them), call .release():
|
||||
//
|
||||
// AvFramePtr nv12{ player->getNV12Frame() };
|
||||
// gpu_frame_attach(mat, nv12.release(), gpuIdx, pts);
|
||||
//
|
||||
// Every .release() in the codebase is a deliberate ownership hand-off —
|
||||
// `git grep '\.release()'` over the migrated files gives an audit list.
|
||||
|
||||
extern "C" {
|
||||
#include <libavutil/frame.h>
|
||||
}
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace ansmedia {
|
||||
|
||||
// Custom deleter that calls av_frame_free (which takes AVFrame**).
|
||||
// noexcept because deleters must not throw — av_frame_free is a C function
|
||||
// and never throws, but tagging makes the contract explicit.
|
||||
struct AvFrameDeleter {
|
||||
void operator()(AVFrame* f) const noexcept {
|
||||
if (f) {
|
||||
av_frame_free(&f); // takes AVFrame**, sets *f to nullptr internally
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Owning smart pointer for AVFrame*. Frees on destruction via av_frame_free.
|
||||
//
|
||||
// Construction:
|
||||
// AvFramePtr p; // empty
|
||||
// AvFramePtr p{ av_frame_clone(src) }; // adopt raw clone
|
||||
//
|
||||
// Ownership transfer (e.g. handing off to gpu_frame_attach_cuda):
|
||||
// takes_ownership(p.release()); // raw pointer escapes, no auto-free
|
||||
//
|
||||
// Copy is disallowed (unique_ptr semantics). Move is fine.
|
||||
using AvFramePtr = std::unique_ptr<AVFrame, AvFrameDeleter>;
|
||||
|
||||
} // namespace ansmedia
|
||||
@@ -10,9 +10,129 @@
|
||||
#include <fstream>
|
||||
#include <cstdlib>
|
||||
#include <system_error>
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
|
||||
namespace ANSCENTER {
|
||||
|
||||
// ====================================================================
|
||||
// DmlSerialState / DmlInferenceGuard implementation.
|
||||
//
|
||||
// Storage is in this single linkage unit (ONNXEngine.dll) so every
|
||||
// module linking against ONNXEngine sees the same flag and mutex —
|
||||
// ANSODEngine.dll's DML calls and ANSOCR.dll's DML calls serialize
|
||||
// against the same process-wide state. See the doc comment on
|
||||
// DmlSerialState in ONNXEngine.h for the full design rationale.
|
||||
// ====================================================================
|
||||
namespace {
|
||||
// false = parallel (default). Tripped to true (sticky) once any
|
||||
// DML site sees an exception or a slow Run().
|
||||
std::atomic<bool> g_dmlSerialMode{ false };
|
||||
|
||||
// Held during DML inference when g_dmlSerialMode == true. Plain
|
||||
// std::mutex (not recursive): one DML Run() at a time across the
|
||||
// entire process — that's the whole point. Other ORT operations
|
||||
// (session creation, allocator setup, etc.) DO NOT need this
|
||||
// mutex and are not gated by it.
|
||||
std::mutex g_dmlInferenceMutex;
|
||||
|
||||
// 1500 ms — comfortably above normal AMD DML latency (50–500 ms)
|
||||
// and well below WDDM TDR (10 s with TdrDelay=10).
|
||||
constexpr double kDmlSlowThresholdMs = 1500.0;
|
||||
}
|
||||
|
||||
bool DmlSerialState::IsSerialMode() {
|
||||
return g_dmlSerialMode.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
double DmlSerialState::SlowThresholdMs() {
|
||||
return kDmlSlowThresholdMs;
|
||||
}
|
||||
|
||||
void DmlSerialState::TripFromException(const char* context, const char* msg) {
|
||||
bool expected = false;
|
||||
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
|
||||
// First caller wins — log the trip event clearly so it's
|
||||
// obvious in the field which path serialized DML.
|
||||
std::cerr << "[DmlSerial] TRIPPED from exception in "
|
||||
<< (context ? context : "?")
|
||||
<< " - process-wide DML inference is now SERIALIZED. msg="
|
||||
<< (msg ? msg : "?") << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void DmlSerialState::TripFromLatency(const char* context, double ms) {
|
||||
bool expected = false;
|
||||
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
|
||||
std::cerr << "[DmlSerial] TRIPPED from slow inference in "
|
||||
<< (context ? context : "?")
|
||||
<< " ms=" << ms
|
||||
<< " - process-wide DML inference is now SERIALIZED."
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void DmlSerialState::AcquireMutex() {
|
||||
g_dmlInferenceMutex.lock();
|
||||
}
|
||||
|
||||
void DmlSerialState::ReleaseMutex() {
|
||||
g_dmlInferenceMutex.unlock();
|
||||
}
|
||||
|
||||
DmlInferenceGuard::DmlInferenceGuard(EngineType ep, const char* context)
|
||||
: _ep(ep), _context(context), _locked(false)
|
||||
{
|
||||
// ── Always-serial-on-AMD policy ────────────────────────────
|
||||
//
|
||||
// Earlier design (Option A v1) was lazy: parallel by default,
|
||||
// serialize only after seeing a slow Run() or exception. Field
|
||||
// testing on AMD Radeon 780M proved this fires too late — by the
|
||||
// time the latency trip arms, a sibling thread has already
|
||||
// entered Run() at depth=2 and wedged the D3D12 command queue.
|
||||
// ORT then crashes inside its own DML EP cleanup when the WDDM
|
||||
// TDR fires.
|
||||
//
|
||||
// The only way to guarantee zero dual-stall on AMD is to never
|
||||
// let two threads be in flight on DML simultaneously. This
|
||||
// constructor unconditionally takes the global DML mutex when
|
||||
// ep == AMD_GPU. Intel (OpenVINO) and NVIDIA (CUDA/TRT) skip
|
||||
// the lock entirely — guard collapses to a timestamp read +
|
||||
// comparison, and the destructor is a single comparison. Zero
|
||||
// measurable overhead on those platforms.
|
||||
//
|
||||
// Trade-off: AMD inference becomes serial (depth=1) instead of
|
||||
// parallel (depth=2). Each individual call still runs on the
|
||||
// GPU at GPU speeds; only the cross-thread overlap is removed.
|
||||
// Throughput cost vs broken depth=2 is ~30% lower wall-clock
|
||||
// throughput per frame, but uptime goes from ~30 s before
|
||||
// wedge → unbounded.
|
||||
if (_ep == EngineType::AMD_GPU) {
|
||||
DmlSerialState::AcquireMutex();
|
||||
_locked = true;
|
||||
}
|
||||
// Start the latency timer AFTER lock acquisition so contention
|
||||
// on the global mutex doesn't itself trip the slow-Run()
|
||||
// threshold. We measure GPU Run() time, not queue wait.
|
||||
_t0 = std::chrono::steady_clock::now();
|
||||
}
|
||||
|
||||
DmlInferenceGuard::~DmlInferenceGuard() {
|
||||
// Latency-based trip detection: if THIS Run() took longer than
|
||||
// the slow threshold, future DML inference goes serial. Only
|
||||
// checked on AMD — Intel/NVIDIA never trip.
|
||||
if (_ep == EngineType::AMD_GPU) {
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double ms = std::chrono::duration<double, std::milli>(t1 - _t0).count();
|
||||
if (ms > kDmlSlowThresholdMs) {
|
||||
DmlSerialState::TripFromLatency(_context, ms);
|
||||
}
|
||||
}
|
||||
if (_locked) {
|
||||
DmlSerialState::ReleaseMutex();
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// BasicOrtHandler — constructors
|
||||
// ====================================================================
|
||||
@@ -365,6 +485,85 @@ namespace ANSCENTER {
|
||||
return false;
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// Reinitialize — tear down + rebuild session on a new EP.
|
||||
//
|
||||
// Used by subclass self-heal paths (ONNXOCRRecognizer /
|
||||
// ONNXOCRDetector) to recover from Ort::Exception in Run(). The
|
||||
// typical flow is:
|
||||
//
|
||||
// try { ort_session->Run(...); }
|
||||
// catch (const Ort::Exception&) {
|
||||
// Reinitialize(EngineType::AMD_GPU); // first attempt: retry DML
|
||||
// try { ort_session->Run(...); }
|
||||
// catch (const Ort::Exception&) {
|
||||
// Reinitialize(EngineType::CPU); // last resort: drop to CPU
|
||||
// ort_session->Run(...);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// After this call returns, ort_session, ort_env, memory_info_handler,
|
||||
// input_node_names, input_node_dims, output_node_names, etc. are all
|
||||
// freshly populated for `newEngine`. Subclass state that lives outside
|
||||
// BasicOrtHandler (e.g. ONNXOCRRecognizer::_warmedUp, ::keys_) is NOT
|
||||
// touched — the caller should reset whatever flags it owns if they
|
||||
// become invalid for the new session.
|
||||
//
|
||||
// Throws Ort::Exception (or std::exception) on failure to construct
|
||||
// the new session. Caller should treat that as terminal for the
|
||||
// current Run() and return an empty result.
|
||||
// ====================================================================
|
||||
void BasicOrtHandler::Reinitialize(EngineType newEngine)
|
||||
{
|
||||
ANS_DBG("OrtHandler",
|
||||
"Reinitialize: oldEngine=%d newEngine=%d (model=%ls)",
|
||||
static_cast<int>(m_engineType),
|
||||
static_cast<int>(newEngine),
|
||||
onnx_path ? onnx_path : L"(null)");
|
||||
|
||||
// Drop the existing session FIRST so its D3D12 / DML allocations
|
||||
// are released back to the driver. Order matters: ort_session
|
||||
// must die before ort_env, otherwise Ort::Session's destructor
|
||||
// can dereference a dangling env pointer in some EP teardown
|
||||
// paths (observed on DML 1.15.x).
|
||||
if (ort_session) {
|
||||
try { delete ort_session; } catch (...) {}
|
||||
ort_session = nullptr;
|
||||
}
|
||||
if (memory_info_handler) {
|
||||
try { delete memory_info_handler; } catch (...) {}
|
||||
memory_info_handler = nullptr;
|
||||
}
|
||||
if (ort_env) {
|
||||
try { delete ort_env; } catch (...) {}
|
||||
ort_env = nullptr;
|
||||
}
|
||||
|
||||
// Reset shape/name vectors so initialize_handler repopulates
|
||||
// them from the new session (input names can differ across EPs
|
||||
// for some models — known DML EP quirk).
|
||||
input_node_names.clear();
|
||||
input_node_names_.clear();
|
||||
input_node_dims.clear();
|
||||
input_values_handler.clear();
|
||||
output_node_names.clear();
|
||||
output_node_names_.clear();
|
||||
output_node_dims.clear();
|
||||
input_name = nullptr;
|
||||
input_tensor_size = 1;
|
||||
num_outputs = 1;
|
||||
|
||||
// Set the requested EP and re-run the standard init. Throws on
|
||||
// failure — caller's responsibility to handle.
|
||||
m_engineType = newEngine;
|
||||
initialize_handler();
|
||||
|
||||
ANS_DBG("OrtHandler",
|
||||
"Reinitialize OK: ort_session=%p engine=%d",
|
||||
static_cast<void*>(ort_session),
|
||||
static_cast<int>(m_engineType));
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// initialize_handler
|
||||
// ====================================================================
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include <typeinfo>
|
||||
#include <deque>
|
||||
#include <unordered_map>
|
||||
#include <chrono> // DmlInferenceGuard latency timing
|
||||
|
||||
#include "onnxruntime_cxx_api.h"
|
||||
#include "opencv2/opencv.hpp"
|
||||
@@ -337,6 +338,108 @@ namespace ANSCENTER {
|
||||
// (AMD fields go here — none yet)
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
// DmlSerialState — process-wide DirectML inference serialization
|
||||
//
|
||||
// AMD Radeon iGPUs (Radeon 680M / 780M, observed) cannot sustain
|
||||
// 2-way concurrent inference on DirectML. Under sustained
|
||||
// parallel load the WDDM driver eventually deadlocks the D3D12
|
||||
// command queue when multiple ORT sessions submit work
|
||||
// simultaneously, producing a "dual stall" pattern (both threads
|
||||
// inside ORT::Run() with no exception thrown) and ultimately an
|
||||
// access violation deep inside the DirectML EP cleanup path.
|
||||
//
|
||||
// Policy: every DirectML inference call on AMD_GPU is serialized
|
||||
// through DmlInferenceGuard's unconditional lock. Lazy "trip on
|
||||
// first slow Run()" was tried first (a flag flipped to true on
|
||||
// exception or >1500 ms latency) but field-proven inadequate —
|
||||
// the trip fires AFTER a slow Run() returns, by which point a
|
||||
// sibling thread has already entered Run() concurrently and
|
||||
// wedged. Always-serialize is the only safe answer on AMD.
|
||||
//
|
||||
// This class still tracks an "ever-tripped" flag for diagnostics
|
||||
// — the TripFromException / TripFromLatency calls log a marker
|
||||
// the first time a DML hiccup is observed, so field traces
|
||||
// clearly show whether the AMD driver had a degraded event. The
|
||||
// flag does NOT gate the lock; the lock is unconditional on AMD.
|
||||
//
|
||||
// Intel (OpenVINO) and NVIDIA (CUDA/TRT) bypass the lock entirely
|
||||
// — DmlInferenceGuard short-circuits on ep != AMD_GPU. They keep
|
||||
// peakDepth=2 parallel inference unchanged; zero behavioral
|
||||
// change on those platforms.
|
||||
//
|
||||
// Storage lives in the ONNXEngine sources compiled into
|
||||
// ANSODEngine.dll so all higher-level modules (ANSODEngine,
|
||||
// ANSOCR, ...) share a single process-global mutex.
|
||||
// ====================================================================
|
||||
class ONNXENGINE_API DmlSerialState {
|
||||
public:
|
||||
// True when serial mode has been tripped.
|
||||
static bool IsSerialMode();
|
||||
|
||||
// Trip helpers. Both are idempotent — only the first caller
|
||||
// per process actually flips the flag and emits the
|
||||
// diagnostic. Called by:
|
||||
// exception path → ANSONNXYOLO::HandleDmlRunFailure, OCR
|
||||
// latency path → DmlInferenceGuard destructor on slow Run()
|
||||
static void TripFromException(const char* context, const char* msg);
|
||||
static void TripFromLatency (const char* context, double ms);
|
||||
|
||||
// 1500 ms — comfortably above normal AMD DML latency and well
|
||||
// under WDDM TDR.
|
||||
static double SlowThresholdMs();
|
||||
|
||||
// Internal — only DmlInferenceGuard should call these.
|
||||
static void AcquireMutex();
|
||||
static void ReleaseMutex();
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
// DmlInferenceGuard — RAII guard for DirectML inference call sites
|
||||
//
|
||||
// Construct around every Ort::Session::Run() that may execute on
|
||||
// a DirectML EP. Behavior:
|
||||
//
|
||||
// 1. ep != AMD_GPU → entire guard is a no-op (zero overhead on
|
||||
// Intel / NVIDIA / CPU). The destructor's latency check and
|
||||
// lock release are also skipped via _ep == AMD_GPU and
|
||||
// _locked == false guards.
|
||||
// 2. ep == AMD_GPU → unconditionally acquires the global DML
|
||||
// mutex; only one DML inference in the entire process runs
|
||||
// at a time. On scope exit, releases the mutex and (if the
|
||||
// call exceeded DmlSerialState::SlowThresholdMs()) records a
|
||||
// diagnostic trip marker — the marker does NOT change
|
||||
// behavior, since AMD is already serial; it just shows up in
|
||||
// logs as evidence of a degraded GPU event.
|
||||
//
|
||||
// Why unconditional serialization on AMD: empirical testing on
|
||||
// Radeon 780M shows the WDDM driver wedges when 2 ORT sessions
|
||||
// submit DML inference concurrently for >~30 s. Lazy "lock only
|
||||
// after first slow call" was found insufficient — by the time the
|
||||
// trip fires, a sibling thread is already inside the wedge-prone
|
||||
// Run() with no way to back out. Always-lock is the only path
|
||||
// that guarantees the dual-stall window never opens. The cost is
|
||||
// ~30% lower throughput on AMD (depth=1 vs depth=2) for
|
||||
// unbounded uptime instead of crashing every 30 s.
|
||||
// ====================================================================
|
||||
class ONNXENGINE_API DmlInferenceGuard {
|
||||
public:
|
||||
explicit DmlInferenceGuard(EngineType ep, const char* context);
|
||||
~DmlInferenceGuard();
|
||||
|
||||
// Non-copyable / non-movable — RAII scope guard only.
|
||||
DmlInferenceGuard(const DmlInferenceGuard&) = delete;
|
||||
DmlInferenceGuard& operator=(const DmlInferenceGuard&) = delete;
|
||||
DmlInferenceGuard(DmlInferenceGuard&&) = delete;
|
||||
DmlInferenceGuard& operator=(DmlInferenceGuard&&) = delete;
|
||||
|
||||
private:
|
||||
EngineType _ep;
|
||||
const char* _context;
|
||||
bool _locked;
|
||||
std::chrono::steady_clock::time_point _t0;
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
// BasicOrtHandler
|
||||
// ====================================================================
|
||||
@@ -410,6 +513,21 @@ namespace ANSCENTER {
|
||||
// (which require the input name) BEFORE the real session is
|
||||
// created. Returns an empty string on failure.
|
||||
static std::string QueryModelInputName(const std::string& onnxPath);
|
||||
protected:
|
||||
// Tear down the existing ORT session/env/memory-info and rebuild
|
||||
// them with `newEngine` as the requested EP. Used by subclass
|
||||
// self-heal paths (e.g. ONNXOCRRecognizer / ONNXOCRDetector) when
|
||||
// an Ort::Exception fires from a Run() — the session is recreated
|
||||
// either on the same EP (transient driver hiccup recovery) or on
|
||||
// CPU (permanent fallback). Throws on failure; caller should
|
||||
// treat that as "give up for this call" and return an empty
|
||||
// result. Subclass-owned auxiliary state (input_node_dims,
|
||||
// input_node_names, ...) is repopulated by initialize_handler.
|
||||
// Caller is responsible for any subclass-specific re-init that
|
||||
// sits on top of the base session (e.g. dictionary loading on
|
||||
// ONNXOCRRecognizer is not affected because it lives in the
|
||||
// subclass and survives a session swap).
|
||||
void Reinitialize(EngineType newEngine);
|
||||
private:
|
||||
void initialize_handler();
|
||||
protected:
|
||||
|
||||
+29
-12
@@ -1,6 +1,7 @@
|
||||
#include "ANSFLV.h"
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameOps.h"
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
|
||||
#include "ANSLicense.h" // ANS_DBG macro
|
||||
#include <memory>
|
||||
@@ -785,17 +786,25 @@ extern "C" __declspec(dllexport) int GetFLVCVImage(ANSCENTER::ANSFLVClient** Han
|
||||
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// NV12 GPU fast path (optional — disabled by default for stability)
|
||||
// NV12 GPU fast path (optional — disabled by default for stability).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
if ((*Handle)->IsNV12FastPath()) {
|
||||
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFLV V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = (*Handle)->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFLV V1 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1032,16 +1041,24 @@ extern "C" __declspec(dllexport) int GetFLVCVImage_V2(uint64_t handleVal, int& w
|
||||
}
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry)
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
int gpuIdx = h->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = h->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = h->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFLV V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = h->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFLV V2 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "ANSFilePlayer.h"
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameOps.h"
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include <memory>
|
||||
#include <cstdint>
|
||||
#include <libswscale/swscale.h>
|
||||
@@ -382,15 +383,24 @@ extern "C" __declspec(dllexport) int GetFilePlayerCVImage(ANSCENTER::ANSFILEPLAY
|
||||
|
||||
// Attach NV12/CUDA frames to registry for inference zero-copy (same as ANSRTSP/ANSVideoPlayer).
|
||||
// CFilePlayer inherits getNV12Frame()/getCudaHWFrame() from CVideoPlayer.
|
||||
// AvFramePtr wraps each clone so it auto-frees on any return path before
|
||||
// the .release() handoff to gpu_frame_attach[_cuda] — leak-by-omission
|
||||
// becomes impossible if an exception or early return slips in later.
|
||||
int gpuIdx = (*Handle)->_playerClient->getHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = (*Handle)->_playerClient->getCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->_playerClient->getCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = (*Handle)->_playerClient->getNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ (*Handle)->_playerClient->getNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFilePlayer V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = (*Handle)->_playerClient->getNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->_playerClient->getNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFilePlayer V1 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
} else {
|
||||
// CPU-only fallback: attach full-res BGR clone for ANSRTYOLO
|
||||
cv::Mat infImg = (*Handle)->GetInferenceImage();
|
||||
@@ -639,16 +649,24 @@ extern "C" __declspec(dllexport) int GetFilePlayerCVImage_V2(uint64_t handleVal,
|
||||
}
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// NV12/CUDA registry attach (same as V1)
|
||||
// NV12/CUDA registry attach (same as V1).
|
||||
// AvFramePtr ensures any clone we obtain is freed automatically if we
|
||||
// don't reach the .release() handoff (e.g. exception, future early return).
|
||||
int gpuIdx = h->_playerClient->getHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = h->_playerClient->getCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ h->_playerClient->getCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = h->_playerClient->getNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ h->_playerClient->getNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFilePlayer V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = h->_playerClient->getNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ h->_playerClient->getNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSFilePlayer V2 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
} else {
|
||||
cv::Mat infImg = h->GetInferenceImage();
|
||||
if (!infImg.empty() && infImg.data != (*image)->data) {
|
||||
|
||||
+29
-12
@@ -1,6 +1,7 @@
|
||||
#include "ANSMJPEG.h"
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameOps.h"
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
|
||||
#include "ANSLicense.h" // ANS_DBG macro
|
||||
#include <memory>
|
||||
@@ -784,17 +785,25 @@ extern "C" __declspec(dllexport) int GetMJPEGCVImage(ANSCENTER::ANSMJPEGClient**
|
||||
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// NV12 GPU fast path (optional — disabled by default for stability)
|
||||
// NV12 GPU fast path (optional — disabled by default for stability).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
if ((*Handle)->IsNV12FastPath()) {
|
||||
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSMJPEG V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = (*Handle)->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSMJPEG V1 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1030,16 +1039,24 @@ extern "C" __declspec(dllexport) int GetMJPEGCVImage_V2(uint64_t handleVal, int&
|
||||
}
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry)
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
int gpuIdx = h->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = h->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = h->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSMJPEG V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = h->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSMJPEG V2 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+29
-12
@@ -1,6 +1,7 @@
|
||||
#include "ANSRTMP.h"
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameOps.h"
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
|
||||
#include "ANSLicense.h" // ANS_DBG macro
|
||||
#include <memory>
|
||||
@@ -808,17 +809,25 @@ extern "C" __declspec(dllexport) int GetRTMPCVImage(ANSCENTER::ANSRTMPClient** H
|
||||
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// NV12 GPU fast path (optional — disabled by default for stability)
|
||||
// NV12 GPU fast path (optional — disabled by default for stability).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
if ((*Handle)->IsNV12FastPath()) {
|
||||
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSRTMP V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = (*Handle)->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSRTMP V1 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1053,16 +1062,24 @@ extern "C" __declspec(dllexport) int GetRTMPCVImage_V2(uint64_t handleVal, int&
|
||||
}
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry)
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
int gpuIdx = h->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = h->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = h->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSRTMP V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = h->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSRTMP V2 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameOps.h"
|
||||
#include "GpuNV12SlotPool.h"
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include "ANSLicense.h" // ANS_DBG macro
|
||||
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
|
||||
#include <memory>
|
||||
@@ -1175,7 +1176,12 @@ extern "C" __declspec(dllexport) int GetRTSPCVImage(
|
||||
(void*)*image, gpuIdx, (int)inFlightGuardHeld);
|
||||
|
||||
if (inFlightGuardHeld) {
|
||||
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
|
||||
// AvFramePtr wraps each clone obtained from the player so it
|
||||
// auto-frees on every return path before the .release() handoff
|
||||
// to gpu_frame_attach[_cuda]. This closes the leak window if
|
||||
// any future early-return / exception slips between the clone
|
||||
// call and the attach call.
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
RTSP_DBG("[GetRTSPCVImage] cudaHW: %dx%d data[0]=%p data[1]=%p",
|
||||
cudaHW->width, cudaHW->height,
|
||||
@@ -1186,13 +1192,20 @@ extern "C" __declspec(dllexport) int GetRTSPCVImage(
|
||||
gpuIdx, cudaHW->width, cudaHW->height);
|
||||
|
||||
// Only fetch CPU NV12 if pool slot unavailable (cross-GPU fallback).
|
||||
AVFrame* cpuNV12 = slot ? nullptr : (*Handle)->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12, slot);
|
||||
ansmedia::AvFramePtr cpuNV12{ slot ? nullptr : (*Handle)->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSRTSP attach_cuda cudaHW=%p cpuNV12=%p slot=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), (void*)slot, gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release(), slot);
|
||||
} else {
|
||||
// HW decode not active — try CPU NV12
|
||||
AVFrame* nv12 = (*Handle)->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSRTSP attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+29
-12
@@ -1,6 +1,7 @@
|
||||
#include "ANSSRT.h"
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameOps.h"
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable()
|
||||
#include "ANSLicense.h" // ANS_DBG macro
|
||||
#include <memory>
|
||||
@@ -826,17 +827,25 @@ extern "C" __declspec(dllexport) int GetSRTCVImage(ANSCENTER::ANSSRTClient** Han
|
||||
// Thread-safe Mat pointer swap (anscv_mat_replace has its own internal lock)
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// NV12 GPU fast path (optional — disabled by default for stability)
|
||||
// NV12 GPU fast path (optional — disabled by default for stability).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
if ((*Handle)->IsNV12FastPath()) {
|
||||
int gpuIdx = (*Handle)->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = (*Handle)->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = (*Handle)->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ (*Handle)->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSSRT V1 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = (*Handle)->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSSRT V1 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1077,16 +1086,24 @@ extern "C" __declspec(dllexport) int GetSRTCVImage_V2(uint64_t handleVal, int& w
|
||||
|
||||
anscv_mat_replace(image, std::move(img));
|
||||
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry)
|
||||
// Attach NV12 frame for GPU fast-path inference (side-table registry).
|
||||
// AvFramePtr auto-frees clones if we don't reach the .release() handoff.
|
||||
int gpuIdx = h->GetHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = h->GetCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ h->GetCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
AVFrame* cpuNV12 = h->GetNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ h->GetNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSSRT V2 attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp,
|
||||
cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = h->GetNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ h->GetNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSSRT V2 attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "ANSMatRegistry.h"
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "ANSGpuFrameOps.h" // gpu_frame_attach, gpu_frame_attach_cuda
|
||||
#include "av_frame_ptr.h" // ansmedia::AvFramePtr — RAII for AVFrame*
|
||||
#include <memory>
|
||||
#include <libswscale/swscale.h>
|
||||
#include <libavutil/imgutils.h>
|
||||
@@ -1010,17 +1011,27 @@ extern "C" __declspec(dllexport) int GetVideoPlayerCVImage(
|
||||
// When using cv::VideoCapture fallback, attach BGR clone (pixFmt=1000).
|
||||
if ((*Handle)->_hwDecodeActive && (*Handle)->_hwPlayer) {
|
||||
// --- HW decode: NV12/CUDA attachment (same pattern as ANSRTSP.cpp:847-862) ---
|
||||
// AvFramePtr wraps each clone so it auto-frees on any return path
|
||||
// before the .release() handoff to gpu_frame_attach[_cuda]. This
|
||||
// makes leak-by-omission impossible if an exception or early return
|
||||
// is added between the clone and the attach call.
|
||||
int gpuIdx = (*Handle)->_hwPlayer->getHWDecodingGpuIndex();
|
||||
AVFrame* cudaHW = (*Handle)->_hwPlayer->getCudaHWFrame();
|
||||
ansmedia::AvFramePtr cudaHW{ (*Handle)->_hwPlayer->getCudaHWFrame() };
|
||||
if (cudaHW) {
|
||||
// CUDA zero-copy: frame data[0]/data[1] are CUDA device pointers.
|
||||
// Also attach CPU NV12 as fallback for cross-GPU inference.
|
||||
AVFrame* cpuNV12 = (*Handle)->_hwPlayer->getNV12Frame();
|
||||
gpu_frame_attach_cuda(*image, cudaHW, gpuIdx, timeStamp, cpuNV12);
|
||||
ansmedia::AvFramePtr cpuNV12{ (*Handle)->_hwPlayer->getNV12Frame() };
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSVideoPlayer attach_cuda cudaHW=%p cpuNV12=%p gpuIdx=%d",
|
||||
(void*)cudaHW.get(), (void*)cpuNV12.get(), gpuIdx);
|
||||
gpu_frame_attach_cuda(*image, cudaHW.release(), gpuIdx, timeStamp, cpuNV12.release());
|
||||
} else {
|
||||
AVFrame* nv12 = (*Handle)->_hwPlayer->getNV12Frame();
|
||||
ansmedia::AvFramePtr nv12{ (*Handle)->_hwPlayer->getNV12Frame() };
|
||||
if (nv12) {
|
||||
gpu_frame_attach(*image, nv12, gpuIdx, timeStamp);
|
||||
ANS_DBG("MEDIA_AvPtr",
|
||||
"ANSVideoPlayer attach nv12=%p gpuIdx=%d",
|
||||
(void*)nv12.get(), gpuIdx);
|
||||
gpu_frame_attach(*image, nv12.release(), gpuIdx, timeStamp);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <filesystem>
|
||||
#include <atomic>
|
||||
#include <opencv2/opencv.hpp>
|
||||
#include "Utility.h"
|
||||
#include "ANSLicense.h"
|
||||
@@ -98,6 +99,31 @@ namespace ANSCENTER
|
||||
bool _debugFlag{ false };
|
||||
bool _licenseValid{ false };
|
||||
bool _isInitialized{ false};
|
||||
|
||||
// Loading-in-progress flag — distinguishes the "LoadEngine() is still
|
||||
// running" state from "LoadEngine() never called or failed". Set by
|
||||
// ModelLoadingGuard at the top of LoadEngine() and cleared on exit
|
||||
// (success, failure, or exception). RunInference/RunInferencesBatch
|
||||
// consult this BEFORE _isInitialized so the early frames during the
|
||||
// 3+ second model-load window don't flood the log with bogus
|
||||
// "Model is not initialized" errors.
|
||||
std::atomic<bool> _modelLoading{ false };
|
||||
|
||||
// RAII helper: sets _modelLoading=true on construction, false on
|
||||
// destruction. Use at the top of LoadEngine() so the flag is always
|
||||
// cleared on every exit path.
|
||||
struct ModelLoadingGuard {
|
||||
std::atomic<bool>& flag;
|
||||
explicit ModelLoadingGuard(std::atomic<bool>& f) : flag(f) {
|
||||
flag.store(true, std::memory_order_release);
|
||||
}
|
||||
~ModelLoadingGuard() {
|
||||
flag.store(false, std::memory_order_release);
|
||||
}
|
||||
ModelLoadingGuard(const ModelLoadingGuard&) = delete;
|
||||
ModelLoadingGuard& operator=(const ModelLoadingGuard&) = delete;
|
||||
};
|
||||
|
||||
double _detectorThreshold{ 0.5 };
|
||||
double _ocrThreshold{ 0.5 };
|
||||
double _colorThreshold{ 0.0 };// If we use the colour information, this threshold should be greater than 0.0
|
||||
|
||||
@@ -221,6 +221,11 @@ namespace ANSCENTER
|
||||
|
||||
bool ANSALPR_OCR::LoadEngine() {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
// RAII guard: set _modelLoading=true for the lifetime of this scope.
|
||||
// RunInference / RunInferencesBatch consult this flag and silently
|
||||
// defer (instead of logging "Model is not initialized") while we're
|
||||
// loading the LP detector + OCR engine + colour classifier.
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
WriteEventLog("ANSALPR_OCR::LoadEngine: Step 1 - Starting engine load");
|
||||
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 1: Starting engine load", __FILE__, __LINE__);
|
||||
@@ -1479,22 +1484,28 @@ namespace ANSCENTER
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInference", "Invalid license", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
if (!_isInitialized) {
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInference", "Model is not initialized", __FILE__, __LINE__);
|
||||
|
||||
// LoadEngine() may be running on another thread (3+ seconds for all
|
||||
// models). Silently defer instead of spamming "Model is not
|
||||
// initialized" — the caller will retry on the next frame.
|
||||
if (_modelLoading.load(std::memory_order_acquire)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Source-of-truth check on the actual engine pointers. If LoadEngine
|
||||
// failed or Destroy() was called, _lpDetector / _ocrEngine are null
|
||||
// and we want a real error logged so the operator sees it.
|
||||
if (!_lpDetector || !_ocrEngine) {
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInference",
|
||||
"Model not loaded (LoadEngine not called or failed)",
|
||||
__FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
if (input.empty() || input.cols < 5 || input.rows < 5) {
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInference", "Input image is empty or too small", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
if (!_lpDetector) {
|
||||
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_lpDetector is null", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
if (!_ocrEngine) {
|
||||
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_ocrEngine is null", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
// Convert grayscale to BGR if necessary
|
||||
@@ -1856,19 +1867,25 @@ namespace ANSCENTER
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", "Invalid license", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
if (!_isInitialized) {
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch", "Model is not initialized", __FILE__, __LINE__);
|
||||
|
||||
// LoadEngine() may be running on another thread (3+ seconds for all
|
||||
// models). Silently defer instead of spamming "Model is not
|
||||
// initialized" — the caller will retry on the next frame.
|
||||
if (_modelLoading.load(std::memory_order_acquire)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Source-of-truth check on the actual engine pointers. If LoadEngine
|
||||
// failed or Destroy() was called, _lpDetector / _ocrEngine are null
|
||||
// and we want a real error logged so the operator sees it.
|
||||
if (!_lpDetector || !_ocrEngine) {
|
||||
this->_logger.LogError("ANSALPR_OCR::RunInferencesBatch",
|
||||
"Model not loaded (LoadEngine not called or failed)",
|
||||
__FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
if (input.empty() || input.cols < 5 || input.rows < 5) return {};
|
||||
if (!_lpDetector) {
|
||||
this->_logger.LogFatal("ANSALPR_OCR::RunInferencesBatch", "_lpDetector is null", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
if (!_ocrEngine) {
|
||||
this->_logger.LogFatal("ANSALPR_OCR::RunInferencesBatch", "_ocrEngine is null", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
if (vehicleBoxes.empty()) return {};
|
||||
|
||||
try {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "include/clipper.h"
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "NV12PreprocessHelper.h"
|
||||
#include "ANSLicense.h" // ANS_DBG
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <iostream>
|
||||
@@ -12,6 +13,130 @@
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — record helper. See
|
||||
// ONNXOCRRecognizer for the rationale. Arms _slowRecreatePending if
|
||||
// a Run() exceeded the slow threshold while past warmup and outside
|
||||
// the cooldown window.
|
||||
void ONNXOCRDetector::RecordRunLatency(const char* context, double ms) {
|
||||
if (ms <= kSlowRecreateThresholdMs) return;
|
||||
if (!_warmedUp) return;
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastRecreateAt).count();
|
||||
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
|
||||
sinceLast < kRecreateCooldownMs) {
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: slow Run (%.0f ms) but in cooldown (%lldms)",
|
||||
context, ms, static_cast<long long>(sinceLast));
|
||||
return;
|
||||
}
|
||||
_slowRecreatePending.store(true, std::memory_order_release);
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": slow Run (" << ms
|
||||
<< " ms > threshold) — flagged for proactive recreate\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
|
||||
context, ms, kSlowRecreateThresholdMs);
|
||||
}
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — apply helper. Called
|
||||
// at entry of public methods. If the slow flag was armed by a prior
|
||||
// call's RecordRunLatency, swap the session in here so this call uses
|
||||
// the fresh one.
|
||||
void ONNXOCRDetector::ApplyPendingSlowRecreate(const char* context) {
|
||||
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
|
||||
return;
|
||||
}
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
|
||||
<< " (latency-triggered)\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: proactive recreate on EP=%d", context,
|
||||
static_cast<int>(currentEp));
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
_lastRecreateAt = std::chrono::steady_clock::now();
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate OK\n";
|
||||
ANS_DBG("ANSONNXDet_run", "%s: proactive recreate OK", context);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate failed: " << e.what() << "\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: proactive recreate failed (%s)", context, e.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate failed (unknown)\n";
|
||||
}
|
||||
}
|
||||
|
||||
// AMD/DirectML self-heal helper. See ONNXOCRRecognizer for the full
|
||||
// rationale. On any Run() failure: recreate session on the same EP
|
||||
// first (transient driver hiccup recovery — the common case for AMD
|
||||
// DML 0x80004005 E_FAIL); on a second failure, fall back to CPU EP.
|
||||
// The current call's result is already lost — caller returns empty;
|
||||
// the recovered session is used by the NEXT Detect call.
|
||||
void ONNXOCRDetector::HandleRunFailureAndRecreate(const char* context,
|
||||
const std::exception& e) {
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
|
||||
<< ": " << e.what() << " — recreating session\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: Run failed EP=%d msg=%.200s — recreating",
|
||||
context, static_cast<int>(currentEp), e.what());
|
||||
|
||||
// Option A: trip process-wide DML serial mode on AMD exception
|
||||
// (idempotent, no-op on Intel/NVIDIA/CPU).
|
||||
if (currentEp == EngineType::AMD_GPU) {
|
||||
ANSCENTER::DmlSerialState::TripFromException(
|
||||
context, e.what());
|
||||
}
|
||||
|
||||
if (currentEp != EngineType::CPU) {
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": session recreated on EP=" << static_cast<int>(currentEp) << "\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: session recreated on EP=%d", context,
|
||||
static_cast<int>(currentEp));
|
||||
return;
|
||||
} catch (const std::exception& reErr) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": same-EP recreate failed: " << reErr.what()
|
||||
<< " — falling back to CPU\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: same-EP recreate failed (%s); falling back to CPU",
|
||||
context, reErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
Reinitialize(EngineType::CPU);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": fallback session created on CPU\n";
|
||||
ANS_DBG("ANSONNXDet_run", "%s: CPU fallback OK", context);
|
||||
} catch (const std::exception& cpuErr) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": CPU fallback ALSO failed: " << cpuErr.what()
|
||||
<< " — ort_session may be null, future calls will return empty\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: CPU fallback failed (%s); ort_session may be null",
|
||||
context, cpuErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": CPU fallback ALSO failed (unknown)\n";
|
||||
}
|
||||
}
|
||||
|
||||
ONNXOCRDetector::ONNXOCRDetector(const std::string& onnx_path, unsigned int num_threads)
|
||||
: BasicOrtHandler(onnx_path, num_threads) {
|
||||
}
|
||||
@@ -55,6 +180,11 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
|
||||
return {};
|
||||
}
|
||||
|
||||
// Path B (AMD/DML): apply pending proactive recreate before constructing
|
||||
// any tensors. See ONNXOCRRecognizer for the full rationale.
|
||||
ApplyPendingSlowRecreate("Detect");
|
||||
if (!ort_session) return {}; // recreate may have failed; bail safely
|
||||
|
||||
// Try to get full-resolution image from NV12 frame (same pattern as ANSONNXYOLO)
|
||||
cv::Mat inferenceImage = srcImage;
|
||||
float bgrScaleX = 1.0f, bgrScaleY = 1.0f;
|
||||
@@ -89,36 +219,60 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
|
||||
*memory_info_handler, inputData.data(), inputData.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
|
||||
// Run inference
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
// Run inference + post-process inside try/catch. AMD/DirectML
|
||||
// self-heal: on Run() failure (typically 0x80004005 E_FAIL from a
|
||||
// transient DML driver hiccup) we recreate the session for the next
|
||||
// call (same EP first, CPU as last resort). This call's result is
|
||||
// lost — return empty boxes; ALPR pipeline tolerates a dropped
|
||||
// detection frame, the tracker recovers. Latency is also recorded
|
||||
// for the Path B proactive-recreate trigger.
|
||||
try {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: serialize DML inference once per-process
|
||||
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType, "ONNXOCRDetector::Detect");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double runMs =
|
||||
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
||||
RecordRunLatency("Detect", runMs);
|
||||
|
||||
// Get output data
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
// Get output data
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
|
||||
int outH = static_cast<int>(outputShape[2]);
|
||||
int outW = static_cast<int>(outputShape[3]);
|
||||
int outH = static_cast<int>(outputShape[2]);
|
||||
int outW = static_cast<int>(outputShape[3]);
|
||||
|
||||
// Postprocess — detection coords are relative to inferenceImage (full-res),
|
||||
// then scaled back to srcImage (display-res) coordinates
|
||||
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
|
||||
inferenceImage.rows, inferenceImage.cols,
|
||||
dbThresh, boxThresh, unclipRatio, useDilation);
|
||||
// Postprocess — detection coords are relative to inferenceImage (full-res),
|
||||
// then scaled back to srcImage (display-res) coordinates
|
||||
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
|
||||
inferenceImage.rows, inferenceImage.cols,
|
||||
dbThresh, boxThresh, unclipRatio, useDilation);
|
||||
|
||||
// Rescale box coordinates from full-res to display-res
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
for (auto& box : boxes) {
|
||||
for (auto& pt : box.points) {
|
||||
pt.x *= bgrScaleX;
|
||||
pt.y *= bgrScaleY;
|
||||
// Rescale box coordinates from full-res to display-res
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
for (auto& box : boxes) {
|
||||
for (auto& pt : box.points) {
|
||||
pt.x *= bgrScaleX;
|
||||
pt.y *= bgrScaleY;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return boxes;
|
||||
return boxes;
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
HandleRunFailureAndRecreate("Detect", e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> ONNXOCRDetector::Preprocess(const cv::Mat& srcImage, int maxSideLen,
|
||||
@@ -323,6 +477,8 @@ void ONNXOCRDetector::Warmup() {
|
||||
// multiple-of-32 rounding. cuDNN caches the algorithm for this
|
||||
// shape so the first real inference doesn't pay the picker cost.
|
||||
constexpr int kWarmupSide = 320;
|
||||
bool warmupFailed = false;
|
||||
std::string warmupErr;
|
||||
try {
|
||||
cv::Mat dummy(kWarmupSide, kWarmupSide, CV_8UC3, cv::Scalar(128, 128, 128));
|
||||
cv::Mat dummyF;
|
||||
@@ -347,6 +503,23 @@ void ONNXOCRDetector::Warmup() {
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
std::cerr << "[ONNXOCRDetector] Warmup failed: " << e.what() << std::endl;
|
||||
warmupFailed = true;
|
||||
warmupErr = e.what();
|
||||
}
|
||||
|
||||
// AMD/DML self-heal: if warmup itself threw, the session is dead and
|
||||
// every Detect() will fail. Recreate now (same EP first, CPU as
|
||||
// fallback) so production traffic can still serve. Leave _warmedUp
|
||||
// = false so the recovered session warms up on its first real call.
|
||||
if (warmupFailed) {
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"Warmup failed (%.200s); triggering self-heal",
|
||||
warmupErr.c_str());
|
||||
try {
|
||||
std::runtime_error proxy(warmupErr);
|
||||
HandleRunFailureAndRecreate("Warmup", proxy);
|
||||
} catch (...) {}
|
||||
return; // do NOT set _warmedUp; recovered session warms on first real Detect
|
||||
}
|
||||
_warmedUp = true;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include "ONNXEngine.h"
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
@@ -53,6 +55,25 @@ private:
|
||||
// Expand 4-point box using Clipper offset
|
||||
std::vector<cv::Point2f> UnclipPolygon(const std::array<cv::Point2f, 4>& box, float unclipRatio);
|
||||
|
||||
// AMD/DirectML self-heal helper — see ONNXOCRRecognizer for full
|
||||
// contract. On Run() failure: recreate session on the same EP first
|
||||
// (transient hiccup recovery), CPU fallback as last resort. Resets
|
||||
// _warmedUp so the new session gets re-warmed on next Detect call.
|
||||
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
|
||||
|
||||
// AMD/DirectML latency-based self-heal (Path B) — see ONNXOCRRecognizer
|
||||
// for the full rationale. Detector is observed to hit the same
|
||||
// slow-degradation pattern as the recognizer after a sibling DML
|
||||
// session takes a fatal: Run() doesn't throw, just gets slow.
|
||||
// 2 s threshold catches degraded individual Run() (healthy is 50–500 ms);
|
||||
// 60 s cooldown prevents thrash on permanently broken driver state.
|
||||
static constexpr double kSlowRecreateThresholdMs = 2000.0;
|
||||
static constexpr int64_t kRecreateCooldownMs = 60000;
|
||||
std::atomic<bool> _slowRecreatePending{false};
|
||||
std::chrono::steady_clock::time_point _lastRecreateAt{};
|
||||
void RecordRunLatency(const char* context, double ms);
|
||||
void ApplyPendingSlowRecreate(const char* context);
|
||||
|
||||
std::mutex _mutex;
|
||||
};
|
||||
|
||||
|
||||
@@ -113,6 +113,165 @@ static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
|
||||
return padded;
|
||||
}
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — record helper.
|
||||
//
|
||||
// Called after every successful production-path Run(). If the call
|
||||
// took longer than kSlowRecreateThresholdMs (5 s default), and the
|
||||
// session has finished its warmup, and we're outside the cooldown
|
||||
// window, arm _slowRecreatePending so the NEXT public method will
|
||||
// perform a session swap before its own Run().
|
||||
//
|
||||
// Doesn't actually do the swap here — that has to happen at a clean
|
||||
// boundary (no in-flight tensors, no held output Ort::Values), which
|
||||
// is precisely the start of the next public method. Calling
|
||||
// Reinitialize from inside the middle of an inference path while
|
||||
// holding output_node_names / inputTensor / outputTensors references
|
||||
// would invalidate them under our feet.
|
||||
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
|
||||
if (ms <= kSlowRecreateThresholdMs) return;
|
||||
if (!_warmedUp) return; // first-shape compile is legitimately slow
|
||||
|
||||
// Cooldown gate — if we recreated recently, hold off. Avoids a
|
||||
// recreate loop if the underlying issue is permanent (e.g. driver
|
||||
// truly broken) rather than a transient hiccup.
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastRecreateAt).count();
|
||||
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
|
||||
sinceLast < kRecreateCooldownMs) {
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
|
||||
context, ms, static_cast<long long>(sinceLast));
|
||||
return;
|
||||
}
|
||||
|
||||
_slowRecreatePending.store(true, std::memory_order_release);
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": slow Run (" << ms
|
||||
<< " ms > threshold) — flagged for proactive session recreate\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
|
||||
context, ms, kSlowRecreateThresholdMs);
|
||||
}
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — apply helper.
|
||||
//
|
||||
// Called at the entry of every public method (Recognize,
|
||||
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
|
||||
// armed by a previous call's RecordRunLatency, do the session swap
|
||||
// here — before any of this call's tensors are constructed. Caller
|
||||
// already holds _mutex, so the new ort_session is exclusively ours
|
||||
// for the rest of this call.
|
||||
//
|
||||
// On Reinitialize failure we leave _slowRecreatePending=false so we
|
||||
// don't loop forever; the existing exception-based HandleRunFailure-
|
||||
// AndRecreate path will fall back to CPU on the next actual throw.
|
||||
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
|
||||
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
|
||||
return;
|
||||
}
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
|
||||
<< " (latency-triggered)\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: proactive recreate on EP=%d (latency-triggered)",
|
||||
context, static_cast<int>(currentEp));
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
_lastRecreateAt = std::chrono::steady_clock::now();
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate succeeded; next call retries on fresh session\n";
|
||||
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate failed: " << e.what()
|
||||
<< " — keeping existing session, exception path will catch real failures\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: proactive recreate failed (%s)", context, e.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate failed (unknown)\n";
|
||||
}
|
||||
}
|
||||
|
||||
// AMD/DirectML self-heal helper. See header comment for detailed contract.
|
||||
// In short: on Run() failure, recreate the session on the same EP first
|
||||
// (transient driver hiccup recovery — the common case); on a second
|
||||
// failure, fall back to CPU EP. Either path leaves ort_session in a
|
||||
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
|
||||
// result is already lost — caller returns empty.
|
||||
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
|
||||
const std::exception& e) {
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
|
||||
<< ": " << e.what() << " — recreating session\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: Run failed EP=%d msg=%.200s — recreating",
|
||||
context, static_cast<int>(currentEp), e.what());
|
||||
|
||||
// Option A: a Run() exception on a DML session is the strongest
|
||||
// signal the AMD driver can't sustain parallel inference. Trip
|
||||
// process-wide DML serial mode so future inference calls (from
|
||||
// any module) take the global mutex. Idempotent — only the
|
||||
// first caller per process flips the flag. No-op on
|
||||
// Intel/NVIDIA/CPU.
|
||||
if (currentEp == EngineType::AMD_GPU) {
|
||||
ANSCENTER::DmlSerialState::TripFromException(
|
||||
context, e.what());
|
||||
}
|
||||
|
||||
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
|
||||
// are transient driver hiccups; a fresh session typically works. This
|
||||
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
|
||||
// throw from Run() under normal load, so this path is rarely taken.
|
||||
if (currentEp != EngineType::CPU) {
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": session recreated on EP=" << static_cast<int>(currentEp)
|
||||
<< " — next call will retry on GPU\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: session recreated on EP=%d (next call retries on GPU)",
|
||||
context, static_cast<int>(currentEp));
|
||||
return;
|
||||
} catch (const std::exception& reCreateErr) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": same-EP recreate failed: " << reCreateErr.what()
|
||||
<< " — falling back to CPU\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: same-EP recreate failed (%s); falling back to CPU",
|
||||
context, reCreateErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: last resort — recreate on CPU. Slower but always works
|
||||
// when the model itself is valid. Caller's next call uses CPU EP.
|
||||
try {
|
||||
Reinitialize(EngineType::CPU);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": fallback session created on CPU\n";
|
||||
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
|
||||
} catch (const std::exception& cpuErr) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": CPU fallback ALSO failed: " << cpuErr.what()
|
||||
<< " — ort_session likely null, future calls will return empty\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: CPU fallback failed (%s); ort_session may be null",
|
||||
context, cpuErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": CPU fallback ALSO failed (unknown exception)\n";
|
||||
}
|
||||
}
|
||||
|
||||
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
|
||||
@@ -120,6 +279,12 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
|
||||
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
|
||||
// are built — so the rest of this method runs on a fresh session.
|
||||
ApplyPendingSlowRecreate("Recognize");
|
||||
if (!ort_session) return {}; // recreate could have failed; bail safely
|
||||
|
||||
try {
|
||||
// Step 1: aspect-preserving resize to height=imgH_, width capped
|
||||
// at imgMaxW_. Then round resized width up to the next bucket.
|
||||
@@ -133,10 +298,24 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
*memory_info_handler, inputData.data(), inputData.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: serialize DML inference once per-process
|
||||
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType, "ONNXOCRRecognizer::Recognize");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double runMs =
|
||||
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
||||
// Latency-based recreate (Path B): arms the flag for the NEXT call
|
||||
// if this one was abnormally slow.
|
||||
RecordRunLatency("Recognize", runMs);
|
||||
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
@@ -147,7 +326,11 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
return CTCDecode(outputData, seqLen, numClasses);
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
std::cerr << "[ONNXOCRRecognizer] Inference failed: " << e.what() << std::endl;
|
||||
// AMD/DML self-heal: recreate session for next call (same EP first,
|
||||
// CPU fallback as last resort). This call's output is lost — return
|
||||
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
|
||||
// tracker migrates plate text across frames.
|
||||
HandleRunFailureAndRecreate("Recognize", e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
@@ -158,6 +341,13 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
std::vector<TextLine>& out) {
|
||||
if (crops.empty()) return;
|
||||
|
||||
// Path B (AMD/DML): apply pending proactive recreate before any tensors
|
||||
// are constructed for this batch. RecognizeBatch holds _mutex around
|
||||
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
|
||||
// same lock the caller already owns.
|
||||
ApplyPendingSlowRecreate("RunBatchAtWidth");
|
||||
if (!ort_session) return; // recreate may have failed; abort safely
|
||||
|
||||
try {
|
||||
// Real number of crops vs. padded batch size submitted to ORT.
|
||||
// Padding to the ladder keeps the recognizer's input-shape set
|
||||
@@ -189,10 +379,22 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
*memory_info_handler, batchInput.data(), batchInput.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: serialize DML inference once per-process
|
||||
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double runMs =
|
||||
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
||||
RecordRunLatency("RunBatchAtWidth", runMs);
|
||||
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
@@ -215,11 +417,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
}
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
// ORT will throw if the model doesn't support a batch dimension > 1.
|
||||
// Fall back to per-image inference for this group.
|
||||
// The original recovery path here covered "model doesn't support
|
||||
// batch dim >1" — a one-shot decision: drop to single-image and
|
||||
// try each crop. We keep that, but ALSO add AMD/DML self-heal
|
||||
// on top: if the per-image fallback ALSO throws, we recreate the
|
||||
// session (same EP first, then CPU) so the next caller gets a
|
||||
// healthy session. The current call's results stay empty for
|
||||
// any crops we couldn't decode — caller tolerates this.
|
||||
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
|
||||
<< bucketW << " (" << e.what()
|
||||
<< ") — falling back to single-image path." << std::endl;
|
||||
bool anyPerImageFailed = false;
|
||||
std::string lastPerImageErr;
|
||||
for (size_t i = 0; i < crops.size(); ++i) {
|
||||
// Direct call (we already hold _mutex via the public RecognizeBatch
|
||||
// wrapper). Replicate the single-image preprocessing here to avoid
|
||||
@@ -232,10 +441,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
|
||||
*memory_info_handler, inputData.data(), inputData.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: per-image fallback path also goes
|
||||
// through the DML serialization gate.
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType,
|
||||
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
float* outData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
int seqLen = static_cast<int>(outShape[1]);
|
||||
@@ -245,6 +462,20 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
|
||||
<< e2.what() << std::endl;
|
||||
out[origIndices[i]] = {};
|
||||
anyPerImageFailed = true;
|
||||
lastPerImageErr = e2.what();
|
||||
}
|
||||
}
|
||||
// If even the per-image fallback path is throwing, the session is
|
||||
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
|
||||
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
|
||||
// session (same EP first, CPU as last resort).
|
||||
if (anyPerImageFailed) {
|
||||
try {
|
||||
std::runtime_error proxy(lastPerImageErr);
|
||||
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
|
||||
} catch (...) {
|
||||
// HandleRunFailureAndRecreate already logs; swallow.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -323,10 +554,13 @@ void ONNXOCRRecognizer::Warmup() {
|
||||
// plates pay a one-time spike on first occurrence of batch=16/24.
|
||||
auto totalT0 = std::chrono::high_resolution_clock::now();
|
||||
int warmedShapes = 0;
|
||||
int totalShapes = 0;
|
||||
std::string lastWarmupErr;
|
||||
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
|
||||
const int batchN = kRecWarmupBatchLadder[bs];
|
||||
for (int b = 0; b < kRecNumBuckets; ++b) {
|
||||
const int bucketW = kRecBucketWidths[b];
|
||||
++totalShapes;
|
||||
try {
|
||||
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
|
||||
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
|
||||
@@ -361,6 +595,7 @@ void ONNXOCRRecognizer::Warmup() {
|
||||
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
|
||||
<< batchN << " bucketW=" << bucketW << ": "
|
||||
<< e.what() << std::endl;
|
||||
lastWarmupErr = e.what();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -368,6 +603,30 @@ void ONNXOCRRecognizer::Warmup() {
|
||||
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
|
||||
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
|
||||
<< " shapes in " << totalMs << " ms total" << std::endl;
|
||||
|
||||
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
|
||||
// production traffic will hit Run() failures on every call. Recreate
|
||||
// it now (same EP first, CPU as fallback) so the pipeline can still
|
||||
// serve. We deliberately keep _warmedUp = false in this path; the
|
||||
// next RecognizeBatch will re-trigger Warmup on the recovered session.
|
||||
//
|
||||
// We do NOT trigger self-heal on partial failure — losing 2 of 16
|
||||
// shapes is normal (e.g. some bucket dims aren't supported by all
|
||||
// EPs); the per-shape catch already guards against that.
|
||||
if (warmedShapes == 0 && totalShapes > 0) {
|
||||
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
|
||||
<< totalShapes << " shapes failed — triggering self-heal\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
|
||||
totalShapes, lastWarmupErr.c_str());
|
||||
try {
|
||||
std::runtime_error proxy(lastWarmupErr);
|
||||
HandleRunFailureAndRecreate("Warmup", proxy);
|
||||
} catch (...) {}
|
||||
// Don't set _warmedUp = true; let the recovered session warm up
|
||||
// itself on the next caller's invocation.
|
||||
return;
|
||||
}
|
||||
_warmedUp = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
@@ -51,6 +53,62 @@ private:
|
||||
// CTC greedy decode
|
||||
TextLine CTCDecode(const float* outputData, int seqLen, int numClasses);
|
||||
|
||||
// AMD/DirectML self-heal helper. Called by Recognize / RunBatchAtWidth /
|
||||
// Warmup when ort_session->Run() throws an Ort::Exception. Tries to
|
||||
// recreate the session on the same EP first (transient driver hiccup
|
||||
// recovery — the common case for DML 0x80004005 E_FAIL); if that fails,
|
||||
// falls back to CPU EP. On full failure leaves ort_session in whatever
|
||||
// state Reinitialize left it (possibly nullptr); the next Recognize
|
||||
// call will short-circuit on the !ort_session check and return empty.
|
||||
// Resets _warmedUp so the new session gets re-warmed on first use.
|
||||
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// AMD/DirectML latency-based self-heal (Path B).
|
||||
//
|
||||
// Observed AMD failure mode: after a transient driver event (e.g.
|
||||
// helmet detector hits DML E_FAIL and CPU-falls-back), DML stays in
|
||||
// a degraded state. OCR Run() calls do NOT throw, but their latency
|
||||
// jumps from 50–200 ms to 1000–7000 ms, often rising over time. The
|
||||
// exception-based self-heal above never fires because there's no
|
||||
// exception. This proactive path observes the latency and forces a
|
||||
// session recreate when a Run() exceeds a sanity threshold — the
|
||||
// fresh session typically returns to baseline latency.
|
||||
//
|
||||
// How it works:
|
||||
// 1. Each public method records its Run() duration.
|
||||
// 2. If duration > kSlowRecreateThresholdMs AND we're past warmup
|
||||
// AND we're past the cooldown window, set _slowRecreatePending.
|
||||
// 3. The NEXT call to a public method checks the flag at entry; if
|
||||
// set, it calls Reinitialize(currentEp) BEFORE its Run() — so
|
||||
// the call that sees the flag uses a fresh session and returns
|
||||
// to fast latency.
|
||||
// 4. After a recreate, _lastRecreateMs starts the cooldown timer.
|
||||
//
|
||||
// Threshold tuning: 2 s catches the AMD-degraded individual Run()
|
||||
// pattern observed in production (1–2 s per OCR Run() when DML is
|
||||
// sick). Healthy individual Run() is 50–500 ms, so 2 s leaves
|
||||
// comfortable headroom. First-shape compiles on OpenVINO can hit
|
||||
// ~1.7 s but those happen during Warmup, which is gated out below
|
||||
// via the _warmedUp check. 60 s cooldown prevents thrash if the
|
||||
// driver is in a permanently bad state — at most one recreate per
|
||||
// minute per session.
|
||||
// ----------------------------------------------------------------
|
||||
static constexpr double kSlowRecreateThresholdMs = 2000.0;
|
||||
static constexpr int64_t kRecreateCooldownMs = 60000;
|
||||
|
||||
// Note: a slow Run() detected by RecordRunLatency arms this flag;
|
||||
// ApplyPendingSlowRecreate (called at every public-method entry)
|
||||
// performs the actual Reinitialize and clears the flag. std::atomic
|
||||
// is used so the read at entry is lock-free; the write happens under
|
||||
// the same _mutex the public methods already hold, so the only race
|
||||
// we're guarding is "another public method picked up the flag first."
|
||||
std::atomic<bool> _slowRecreatePending{false};
|
||||
std::chrono::steady_clock::time_point _lastRecreateAt{};
|
||||
|
||||
void RecordRunLatency(const char* context, double ms);
|
||||
void ApplyPendingSlowRecreate(const char* context);
|
||||
|
||||
std::vector<std::string> keys_;
|
||||
int imgH_ = kRecImgH;
|
||||
int imgMaxW_ = kRecImgMaxW;
|
||||
|
||||
@@ -24,7 +24,7 @@ namespace ANSCENTER {
|
||||
std::unique_ptr<IANSCustomClass> _customDetector;
|
||||
HMODULE hMod= nullptr;
|
||||
std::string FindANSCustomLibraryName(const std::string& folderPath, const std::string& keyword);
|
||||
bool CreateCustomDetector();
|
||||
bool CreateCustomDetector();
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@@ -36,6 +36,57 @@ const int MAX_MISSING_SCREEN = 1000;
|
||||
|
||||
namespace ANSCENTER
|
||||
{
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// DirectML fatal-error detection
|
||||
//
|
||||
// AMD's DirectML EP can fail in several distinct ways that all leave the
|
||||
// Ort::Session permanently broken and require a fresh CPU-fallback session:
|
||||
//
|
||||
// 887A0005 DXGI_ERROR_DEVICE_REMOVED — adapter killed (RDP attach,
|
||||
// TDR, sleep/wake, driver swap). GetDeviceRemovedReason gives
|
||||
// the sub-cause but the device is gone either way.
|
||||
// 80004005 E_FAIL — generic D3D12 / DML failure. Seen from
|
||||
// DmlGraphFusionHelper::ExecuteOperator and
|
||||
// DmlCommandRecorder::ExecuteCommandList on AMD when the
|
||||
// driver couldn't compile/execute a fused subgraph or when
|
||||
// TDR fires mid-command-list. No structured sub-cause
|
||||
// available — must treat as device-fatal.
|
||||
//
|
||||
// Branding strings ("DmlFusedNode", "DmlGraphFusionHelper",
|
||||
// "DmlCommandRecorder") catch future failure modes that surface the same
|
||||
// way without us having to chase each new HRESULT.
|
||||
//
|
||||
// Used by every catch site that wraps Ort::Session::Run on AMD DML.
|
||||
// Centralised here so a new failure mode is handled once, not 4+ times.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
inline bool IsDmlFatalError(const std::string& msg) {
|
||||
return msg.find("887A0005") != std::string::npos
|
||||
|| msg.find("80004005") != std::string::npos
|
||||
|| msg.find("DmlFusedNode") != std::string::npos
|
||||
|| msg.find("DmlGraphFusionHelper") != std::string::npos
|
||||
|| msg.find("DmlCommandRecorder") != std::string::npos
|
||||
|| msg.find("DmlExecutionProvider") != std::string::npos;
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// IsDmlDeviceRemoved — narrowly checks ONLY 887A0005
|
||||
// (DXGI_ERROR_DEVICE_REMOVED). This is the truly unrecoverable case:
|
||||
// the D3D12 adapter is gone (driver crashed, RDP attach killed the
|
||||
// device, sleep/wake torched the WDDM context). No amount of session
|
||||
// recreation will bring it back — only a process restart or CPU
|
||||
// fallback can recover.
|
||||
//
|
||||
// In contrast, 80004005 (E_FAIL) from DmlGraphFusionHelper /
|
||||
// DmlCommandRecorder is intermittent on AMD when the driver chokes on
|
||||
// a fused subgraph or TDR fires mid-command-list. A fresh DML
|
||||
// session typically clears it and resumes GPU inference — that's the
|
||||
// stay-on-GPU recovery path used by ANSONNXYOLO's catch blocks.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
inline bool IsDmlDeviceRemoved(const std::string& msg) {
|
||||
return msg.find("887A0005") != std::string::npos
|
||||
|| msg.find("DXGI_ERROR_DEVICE_REMOVED") != std::string::npos;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
typename std::enable_if<std::is_arithmetic<T>::value, T>::type
|
||||
inline clamp(const T& value, const T& low, const T& high)
|
||||
|
||||
+113
-197
@@ -1,4 +1,5 @@
|
||||
#include"ANSONNXCL.h"
|
||||
#include "ANSONNXYOLO.h" // ANSONNXYOLO is the delegate target — see facade comment in ANSONNXCL.h
|
||||
#include "EPLoader.h"
|
||||
#include "OpenVINODeviceConfig.h"
|
||||
#include <typeinfo> // typeid(e).name() in ANS_DBG diagnostics
|
||||
@@ -876,9 +877,50 @@ namespace ANSCENTER
|
||||
);
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
const std::string msg = e.what();
|
||||
this->_logger.LogError("ANSONNXCL::classify",
|
||||
"ONNX Runtime Exception during Run(): " + std::string(e.what()), __FILE__, __LINE__);
|
||||
"ONNX Runtime Exception during Run(): " + msg, __FILE__, __LINE__);
|
||||
delete[] blobPtr; // Clean up on error
|
||||
blobPtr = nullptr;
|
||||
|
||||
// DML device-lost / DML graph-execution failure → tear down the
|
||||
// dead DirectML session and rebuild on CPU. Without this, every
|
||||
// subsequent classify() call hits the same dead session and
|
||||
// floods the event log (16k+ entries observed on AMD/RDP).
|
||||
if (IsDmlFatalError(msg) && !_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXCL::classify",
|
||||
std::string("DirectML fatal error — recreating session on CPU. msg=")
|
||||
+ msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSONNXCL", "DML fatal — rebuild on CPU msg=%s", msg.c_str());
|
||||
try {
|
||||
// Order matters: drop AllocatedStringPtrs (which depend
|
||||
// on the live session's allocator) BEFORE tearing down
|
||||
// session_, sessionOptions_, env_. Init() will repopulate.
|
||||
inputNodeNameAllocatedStrings_.clear();
|
||||
outputNodeNameAllocatedStrings_.clear();
|
||||
inputNames_.clear();
|
||||
outputNames_.clear();
|
||||
session_ = Ort::Session{ nullptr };
|
||||
sessionOptions_ = Ort::SessionOptions{ nullptr };
|
||||
env_ = Ort::Env{ nullptr };
|
||||
if (Init(_modelFilePath, inputImageShape_, /*useGPU=*/false)) {
|
||||
_logger.LogInfo("ANSONNXCL::classify",
|
||||
"CPU fallback session created successfully",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSONNXCL", "CPU fallback OK");
|
||||
} else {
|
||||
_logger.LogFatal("ANSONNXCL::classify",
|
||||
"CPU fallback session creation failed",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
} catch (const std::exception& re) {
|
||||
_logger.LogFatal("ANSONNXCL::classify",
|
||||
std::string("CPU fallback exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -922,216 +964,90 @@ namespace ANSCENTER
|
||||
return {};
|
||||
}
|
||||
}
|
||||
bool ANSONNXCL::OptimizeModel(bool fp16, std::string & optimizedModelFolder) {
|
||||
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
_modelLoadValid = false;
|
||||
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
|
||||
if (!result) return false;
|
||||
// Parsing for YOLO only here
|
||||
_modelConfig = modelConfig;
|
||||
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
|
||||
_modelConfig.modelType = ModelType::ONNXCL;
|
||||
_modelConfig.inpHeight = 224;
|
||||
_modelConfig.inpWidth = 224;
|
||||
if (_modelConfig.modelMNSThreshold < 0.2)
|
||||
_modelConfig.modelMNSThreshold = 0.5;
|
||||
if (_modelConfig.modelConfThreshold < 0.2)
|
||||
_modelConfig.modelConfThreshold = 0.5;
|
||||
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
|
||||
_modelConfig.numKPS = 17;
|
||||
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
|
||||
_fp16 = (modelConfig.precisionType == PrecisionType::FP16);
|
||||
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// Public-method facade: delegate everything to the wrapped ANSONNXYOLO.
|
||||
//
|
||||
// This is a verification harness — ANSONNXYOLO already covers the
|
||||
// classification path (auto-detected from model output shape) and ships
|
||||
// with the safe DML config + DML-fatal-error recovery. By routing every
|
||||
// ANSONNXCL public call through ANSONNXYOLO we can A/B test whether
|
||||
// ANSONNXCL's own session/postprocess logic is the actual cause of the
|
||||
// AMD same-process DML races we saw with the custom helmet model.
|
||||
//
|
||||
// The dead code below the facade (Init / preprocess / postprocess /
|
||||
// classify / Ort::* members in the header) is intentionally preserved
|
||||
// so the original implementation is one git-revert away if needed.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
namespace {
|
||||
// Lazy-create the delegate on first public call. Not thread-safe in
|
||||
// the strict sense, but the public methods all take _mutex first via
|
||||
// the underlying ANSODBase, so concurrent first-time creation is
|
||||
// serialised in practice.
|
||||
inline ANSONNXYOLO& GetDelegate(std::unique_ptr<ANSONNXYOLO>& slot) {
|
||||
if (!slot) {
|
||||
slot = std::make_unique<ANSONNXYOLO>();
|
||||
ANS_DBG("ANSONNXCL_Facade",
|
||||
"delegate created (ANSONNXYOLO=%p)", (void*)slot.get());
|
||||
}
|
||||
else {// This is old version of model zip file
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValidFileName(_classFilePath);
|
||||
if (!isValidFileName)
|
||||
{
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromString();
|
||||
}
|
||||
else {
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
// 1. Load labelMap and engine
|
||||
labelMap.clear();
|
||||
if (!_classes.empty())
|
||||
labelMap = VectorToCommaSeparatedString(_classes);
|
||||
|
||||
// 2. Initialize ONNX Runtime session
|
||||
cv::Size targetInputShape= cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
|
||||
result = Init(_modelFilePath, targetInputShape,true);
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::Initialize", e.what(), __FILE__, __LINE__);
|
||||
return false;
|
||||
return *slot;
|
||||
}
|
||||
}
|
||||
bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
|
||||
if (!result) return false;
|
||||
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
|
||||
_modelConfig.modelType = ModelType::TENSORRT;
|
||||
_modelConfig.inpHeight = 224;
|
||||
_modelConfig.inpWidth = 224;
|
||||
if (_modelConfig.modelMNSThreshold < 0.2)
|
||||
_modelConfig.modelMNSThreshold = 0.5;
|
||||
if (_modelConfig.modelConfThreshold < 0.2)
|
||||
_modelConfig.modelConfThreshold = 0.5;
|
||||
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
|
||||
_modelConfig.numKPS = 17;
|
||||
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
|
||||
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
|
||||
_fp16 = true; // Load Model from Here
|
||||
|
||||
// 0. Check if the configuration file exist
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
}
|
||||
else {// This is old version of model zip file
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValidFileName(_classFilePath);
|
||||
if (!isValidFileName)
|
||||
{
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromString();
|
||||
}
|
||||
else {
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
// Initialize ONNX Runtime session
|
||||
cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
|
||||
result = Init(_modelFilePath, targetInputShape,true);
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::LoadModel", e.what(), __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
bool ANSONNXCL::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "OptimizeModel → ANSONNXYOLO");
|
||||
return GetDelegate(_yoloDelegate).OptimizeModel(fp16, optimizedModelFolder);
|
||||
}
|
||||
bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
|
||||
if (!result) return false;
|
||||
std::string _modelName = modelName;
|
||||
if (_modelName.empty()) {
|
||||
_modelName = "train_last";
|
||||
}
|
||||
std::string modelFullName = _modelName + ".onnx";
|
||||
// Parsing for YOLO only here
|
||||
_modelConfig = modelConfig;
|
||||
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
|
||||
_modelConfig.modelType = ModelType::TENSORRT;
|
||||
_modelConfig.inpHeight = 224;
|
||||
_modelConfig.inpWidth = 224;
|
||||
if (_modelConfig.modelMNSThreshold < 0.2)
|
||||
_modelConfig.modelMNSThreshold = 0.5;
|
||||
if (_modelConfig.modelConfThreshold < 0.2)
|
||||
_modelConfig.modelConfThreshold = 0.5;
|
||||
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
|
||||
_modelConfig.numKPS = 17;
|
||||
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
|
||||
_fp16 = true; // Load Model from Here
|
||||
|
||||
// 0. Check if the configuration file exist
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
}
|
||||
else {// This is old version of model zip file
|
||||
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
|
||||
_classFilePath = CreateFilePath(_modelFolder, className);
|
||||
std::ifstream isValidFileName(_classFilePath);
|
||||
if (!isValidFileName)
|
||||
{
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromString();
|
||||
}
|
||||
else {
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
// 1. Load labelMap and engine
|
||||
labelMap.clear();
|
||||
if (!_classes.empty())
|
||||
labelMap = VectorToCommaSeparatedString(_classes);
|
||||
// 2. Initialize ONNX Runtime session
|
||||
cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
|
||||
result = Init(_modelFilePath, targetInputShape,true);
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||
const std::string& modelZipFilePath,
|
||||
const std::string& modelZipPassword,
|
||||
std::string& labelMap) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "Initialize → ANSONNXYOLO zip=%s",
|
||||
modelZipFilePath.c_str());
|
||||
return GetDelegate(_yoloDelegate).Initialize(
|
||||
licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
|
||||
}
|
||||
|
||||
bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath,
|
||||
const std::string& modelZipPassword) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "LoadModel → ANSONNXYOLO zip=%s",
|
||||
modelZipFilePath.c_str());
|
||||
return GetDelegate(_yoloDelegate).LoadModel(modelZipFilePath, modelZipPassword);
|
||||
}
|
||||
|
||||
bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
|
||||
std::string modelName, std::string className,
|
||||
const std::string& modelFolder,
|
||||
std::string& labelMap) {
|
||||
ANS_DBG("ANSONNXCL_Facade",
|
||||
"LoadModelFromFolder → ANSONNXYOLO modelName=%s folder=%s",
|
||||
modelName.c_str(), modelFolder.c_str());
|
||||
return GetDelegate(_yoloDelegate).LoadModelFromFolder(
|
||||
licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
|
||||
}
|
||||
|
||||
bool ANSONNXCL::Destroy() {
|
||||
return true;
|
||||
if (_yoloDelegate) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "Destroy → ANSONNXYOLO");
|
||||
return _yoloDelegate->Destroy();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
ANSONNXCL::~ANSONNXCL() {
|
||||
// Explicit Destroy() so the delegate's session is torn down before
|
||||
// the unique_ptr destructor frees the object. unique_ptr<ANSONNXYOLO>
|
||||
// still releases the heap memory afterwards.
|
||||
Destroy();
|
||||
}
|
||||
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& input, const std::string& camera_id) {
|
||||
if (!PreInferenceCheck("ANSONNXCL::RunInference")) return {};
|
||||
try {
|
||||
std::vector<Object> result;
|
||||
if (input.empty()) return result;
|
||||
if ((input.cols < 5) || (input.rows < 5)) return result;
|
||||
return classify(input, camera_id);
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::RunInference", e.what(), __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& input,
|
||||
const std::string& camera_id) {
|
||||
if (input.empty()) return {};
|
||||
if (input.cols < 5 || input.rows < 5) return {};
|
||||
return GetDelegate(_yoloDelegate).RunInference(input, camera_id);
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& inputImgBGR) {
|
||||
return RunInference(inputImgBGR, "CustomCam");
|
||||
}
|
||||
|
||||
@@ -3,8 +3,26 @@
|
||||
#pragma once
|
||||
#include "ANSEngineCommon.h"
|
||||
#include "engine.h"
|
||||
#include <memory>
|
||||
|
||||
namespace ANSCENTER {
|
||||
// Forward declaration: avoid pulling ANSONNXYOLO.h into every consumer of
|
||||
// ANSONNXCL.h. Definition is included in ANSONNXCL.cpp.
|
||||
class ANSONNXYOLO;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// ANSONNXCL — verification facade over ANSONNXYOLO.
|
||||
//
|
||||
// ANSONNXYOLO already auto-detects classification models (single 2D
|
||||
// output [B, nc]) and routes through its own classification postprocess.
|
||||
// It also has the safe DML config (DisableMemPattern + ORT_SEQUENTIAL +
|
||||
// EXTENDED) and the IsDmlFatalError catch-and-rebuild path.
|
||||
//
|
||||
// We delegate every public method here to an internal ANSONNXYOLO so we
|
||||
// can A/B test whether ANSONNXCL's own session/postprocess code is the
|
||||
// real cause of the AMD same-process DML races we saw with the custom
|
||||
// helmet model. Public ABI is unchanged — external callers don't move.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
class ANSENGINE_API ANSONNXCL :public ANSODBase {
|
||||
public:
|
||||
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
|
||||
@@ -15,10 +33,23 @@ namespace ANSCENTER {
|
||||
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
|
||||
bool Destroy();
|
||||
~ANSONNXCL();
|
||||
private:
|
||||
// The wrapped engine. Lazy-initialised on first public call so we
|
||||
// don't pay the cost (and pull in the dependency) for callers who
|
||||
// never use this class.
|
||||
std::unique_ptr<ANSONNXYOLO> _yoloDelegate;
|
||||
private:
|
||||
std::string _modelFilePath;
|
||||
bool _modelLoadValid;
|
||||
bool _fp16{ false };
|
||||
|
||||
// DML device-lost recovery: when DirectML fails fatally
|
||||
// (HRESULT 887A0005 device-removed, 80004005 graph/command-recorder
|
||||
// E_FAIL, etc. — see ANSCENTER::IsDmlFatalError) the Ort::Session
|
||||
// is permanently broken. We detect once, recreate on CPU via
|
||||
// Init(_modelFilePath, inputImageShape_, /*useGPU=*/false), and
|
||||
// suppress further error-log flooding via this flag.
|
||||
bool _dmlDeviceLost{ false };
|
||||
size_t vectorProduct(const std::vector<int64_t>& vector);
|
||||
bool preprocessImageToTensor(const cv::Mat& image, cv::Mat& outImage,
|
||||
const cv::Size& targetShape,
|
||||
|
||||
@@ -1610,7 +1610,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath);
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1626,7 +1626,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath, engineType);
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath, engineType);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1638,26 +1638,35 @@ namespace ANSCENTER {
|
||||
// ========================================================================
|
||||
// WarmUpEngine — run a dummy inference after session creation.
|
||||
//
|
||||
// Scope: **NVIDIA (CUDA EP) only.** On first inference, the CUDA EP
|
||||
// allocates its memory arena (capped at 2 GB via BasicOrtHandler config),
|
||||
// resolves cuDNN convolution algorithms, and populates the kernel launch
|
||||
// cache. Running one dummy inference at load time amortises this cost
|
||||
// so the first real frame doesn't see a latency spike.
|
||||
// Scope: NVIDIA (CUDA EP) AND AMD (DirectML EP).
|
||||
// • NVIDIA: amortises CUDA arena allocation, cuDNN convolution algo
|
||||
// selection, and kernel-launch cache population.
|
||||
// • AMD: amortises D3D12 PSO compilation and first-time DML kernel
|
||||
// binding. Empirically ~150 ms on DirectML (see comment in
|
||||
// ONNXOCRRecognizer::Warmup). Without this, that compile cost
|
||||
// surfaces inside the first live Run() — which on AMD has been
|
||||
// observed to race with concurrent DML traffic from other sessions
|
||||
// (e.g. helmet detector running while ALPR's _lpDetector fires its
|
||||
// first ever Run()) and trigger DmlGraphFusionHelper crashes /
|
||||
// 887A0006 DEVICE_HUNG. Moving the heavy first-run work into
|
||||
// LoadEngine — where only the loading thread is active — sidesteps
|
||||
// this timing window.
|
||||
//
|
||||
// Explicitly disabled on AMD, Intel and CPU:
|
||||
// • AMD (DirectML) — calling detect() at load time has been observed
|
||||
// to hit a multi-terabyte cv::Mat allocation inside postprocessLegacy
|
||||
// on AMD RDNA iGPUs when DirectML returns garbage output tensor
|
||||
// dims. ONNXYOLO::detect() now has an output-shape sanity guard
|
||||
// that catches this at runtime, so the warm-up would add risk
|
||||
// without benefit. Earlier builds enabled warm-up specifically for
|
||||
// Radeon 680M TDR mitigation; that workaround is obsolete with
|
||||
// current DirectML 1.15.x drivers.
|
||||
// • Intel (OpenVINO) — running detect() at load time has been
|
||||
// observed to expose latent heap-corruption bugs
|
||||
// (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
|
||||
// • CPU EP — no shader compile or kernel cache to warm up; the first
|
||||
// real frame has the same latency as any subsequent frame.
|
||||
// Skipped for:
|
||||
// • Intel (OpenVINO EP) — OpenVINO compiles the full plan at session
|
||||
// creation, so first Run() has no extra compile cost. Calling
|
||||
// detect() at load time has also exposed latent heap-corruption
|
||||
// bugs (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
|
||||
// • CPU EP — no shader / kernel cache to warm up.
|
||||
//
|
||||
// History note: AMD warm-up was disabled in earlier builds after
|
||||
// observing multi-terabyte cv::Mat allocations from garbage DirectML
|
||||
// output dims. ONNXYOLO::detect() now has an output-shape sanity
|
||||
// guard (see line ~257, kMaxOutputDim = 1,000,000) that catches that
|
||||
// case before postprocessLegacy can allocate, so re-enabling AMD
|
||||
// warm-up is safe. The exception-swallowing catch blocks below
|
||||
// guarantee that any warm-up failure is non-fatal — the session is
|
||||
// already loaded and real inference will still work.
|
||||
//
|
||||
// Non-fatal on failure: if warm-up itself throws, regular inference
|
||||
// still works — the engine is fully loaded before WarmUpEngine runs.
|
||||
@@ -1665,9 +1674,11 @@ namespace ANSCENTER {
|
||||
void ANSONNXYOLO::WarmUpEngine() {
|
||||
if (!m_ortEngine) return;
|
||||
|
||||
// Gate strictly on NVIDIA_GPU. Every other EP is a no-op.
|
||||
if (m_ortEngine->getEngineType() != EngineType::NVIDIA_GPU) {
|
||||
ANS_DBG("ONNXYOLO", "Warm-up skipped (non-NVIDIA EP)");
|
||||
// Warm up on NVIDIA and AMD (both have lazy first-Run() compile cost).
|
||||
// Skip on CPU and OpenVINO — see scope comment above.
|
||||
const auto et = m_ortEngine->getEngineType();
|
||||
if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) {
|
||||
ANS_DBG("ONNXYOLO", "Warm-up skipped (CPU/OpenVINO EP)");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1696,7 +1707,8 @@ namespace ANSCENTER {
|
||||
// preprocessing (114,114,114 ~ 128) and avoids degenerate inputs.
|
||||
cv::Mat dummy(h, w, CV_8UC3, cv::Scalar(128, 128, 128));
|
||||
|
||||
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy CUDA inference (%dx%d)", w, h);
|
||||
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy %s inference (%dx%d)",
|
||||
(et == EngineType::NVIDIA_GPU) ? "CUDA" : "DirectML", w, h);
|
||||
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
(void)m_ortEngine->detect(dummy, _classes,
|
||||
@@ -1975,6 +1987,150 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// HandleDmlRunFailure — stay-on-DML recovery (replaces CPU-fallback).
|
||||
//
|
||||
// Called from every catch block that wraps an Ort::Session::Run on a
|
||||
// DirectML EP. The contract is documented on the member declaration
|
||||
// in ANSONNXYOLO.h — quick summary:
|
||||
//
|
||||
// * CAS on _dmlRecreateInProgress elects ONE recreate owner per
|
||||
// driver hiccup; other concurrent failures just bail.
|
||||
// * Cooldown (kDmlRecreateCooldownMs) prevents thrashing if E_FAIL
|
||||
// keeps firing back to back.
|
||||
// * IsDmlDeviceRemoved (887A0005) → CPU fallback, last resort.
|
||||
// * After kMaxConsecutiveDmlFailures recreates without an
|
||||
// intervening successful inference → CPU fallback.
|
||||
// * Otherwise: reset() + InitOrtEngine(<original DML EP>) +
|
||||
// WarmUpEngine() → fresh DML session, GPU inference resumes.
|
||||
//
|
||||
// Successful inferences reset _dmlConsecutiveFailures to 0 in
|
||||
// DetectObjects / DetectObjectsBatch, so isolated transient hiccups
|
||||
// do not accumulate toward the CPU-fallback threshold.
|
||||
// ====================================================================
|
||||
void ANSONNXYOLO::HandleDmlRunFailure(const char* context, const std::string& msg)
|
||||
{
|
||||
// Option A: a DML exception is the strongest possible signal
|
||||
// that the AMD driver can't sustain parallel inference. Trip
|
||||
// process-wide DML serial mode immediately so the NEXT
|
||||
// inference attempt (from any thread, in this DLL or another)
|
||||
// takes the global mutex and runs one-at-a-time. This is
|
||||
// idempotent — only the first caller per process actually
|
||||
// flips the flag. Done before the CAS so even the CAS losers
|
||||
// (concurrent threads catching the same fatal) contribute the
|
||||
// signal — but in practice the first one wins.
|
||||
ANSCENTER::DmlSerialState::TripFromException(context, msg.c_str());
|
||||
|
||||
// CAS — only one thread owns the recreate at a time. Other
|
||||
// concurrent DML failures just return and retry on the next call.
|
||||
bool expected = false;
|
||||
if (!_dmlRecreateInProgress.compare_exchange_strong(expected, true)) {
|
||||
ANS_DBG("ONNXYOLO", "%s: DML fatal — another thread is recreating, skip",
|
||||
context);
|
||||
return;
|
||||
}
|
||||
|
||||
// Hold the engine mutex for the entire recreate so concurrent
|
||||
// inference threads (which use a shared_ptr local copy of the
|
||||
// engine) don't see a half-built session.
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
|
||||
try {
|
||||
// Cooldown — if we just recreated within the window, the
|
||||
// driver is in a prolonged bad state; don't burn time
|
||||
// recreating again immediately. Other threads will keep
|
||||
// returning {} until either the next call past cooldown
|
||||
// succeeds or we hit the CPU-fallback threshold.
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLastMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastDmlRecreateAt).count();
|
||||
if (_lastDmlRecreateAt.time_since_epoch().count() != 0
|
||||
&& sinceLastMs < kDmlRecreateCooldownMs) {
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"%s: DML fatal — within cooldown (%lld ms < %lld), skip",
|
||||
context, (long long)sinceLastMs, (long long)kDmlRecreateCooldownMs);
|
||||
_dmlRecreateInProgress.store(false);
|
||||
return;
|
||||
}
|
||||
_lastDmlRecreateAt = now;
|
||||
|
||||
// Decide target EP for the new session:
|
||||
// * 887A0005 device-removed → CPU is the only option
|
||||
// * Too many consecutive failures → CPU as last resort
|
||||
// * Otherwise → recreate on the SAME DML EP we were on
|
||||
ANSCENTER::EngineType originalEp = ANSCENTER::EngineType::CPU;
|
||||
if (m_ortEngine) {
|
||||
originalEp = m_ortEngine->getEngineType();
|
||||
}
|
||||
|
||||
const int failures = _dmlConsecutiveFailures.fetch_add(1) + 1;
|
||||
const bool deviceRemoved = ANSCENTER::IsDmlDeviceRemoved(msg);
|
||||
const bool exceededRetries = failures > kMaxConsecutiveDmlFailures;
|
||||
|
||||
ANSCENTER::EngineType targetEp = originalEp;
|
||||
const char* reason = "stay-on-DML retry";
|
||||
if (deviceRemoved) {
|
||||
targetEp = ANSCENTER::EngineType::CPU;
|
||||
reason = "device-removed (887A0005) → CPU fallback";
|
||||
} else if (exceededRetries) {
|
||||
targetEp = ANSCENTER::EngineType::CPU;
|
||||
reason = "exceeded retry budget → CPU fallback";
|
||||
}
|
||||
|
||||
_logger.LogFatal(context,
|
||||
std::string("DirectML fatal — ") + reason
|
||||
+ " (failure #" + std::to_string(failures)
|
||||
+ "). msg=" + msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"%s: DML fatal recreate — %s, failures=%d, originalEp=%d, targetEp=%d",
|
||||
context, reason, failures, (int)originalEp, (int)targetEp);
|
||||
|
||||
// reset() releases the member's reference; concurrent inference
|
||||
// threads holding a shared_ptr local copy keep their old
|
||||
// engine alive until they return — no use-after-free (B2).
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(targetEp)) {
|
||||
_logger.LogFatal(context,
|
||||
"DML recreate failed — m_ortEngine left null", __FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "%s: InitOrtEngine(%d) failed", context, (int)targetEp);
|
||||
_dmlRecreateInProgress.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Warm up the new session so the first real call doesn't pay
|
||||
// the algorithm-selection / shader-compile tax. AMD DML
|
||||
// recreate WITHOUT warmup typically hits 1-2 s on the very
|
||||
// first inference, which can re-trigger our 2 s slow watchdog
|
||||
// and produce a false alarm.
|
||||
try {
|
||||
WarmUpEngine();
|
||||
} catch (const std::exception& we) {
|
||||
ANS_DBG("ONNXYOLO", "%s: WarmUpEngine after recreate threw: %s",
|
||||
context, we.what());
|
||||
}
|
||||
|
||||
// If we landed on CPU as a last resort, leave the failure
|
||||
// counter pinned so we don't oscillate back to DML on the
|
||||
// next exception. Otherwise (we're back on DML) DON'T reset
|
||||
// the counter here — it gets reset by a *successful*
|
||||
// inference in DetectObjects / DetectObjectsBatch.
|
||||
if (targetEp == ANSCENTER::EngineType::CPU
|
||||
&& originalEp != ANSCENTER::EngineType::CPU) {
|
||||
_logger.LogInfo(context,
|
||||
"Permanently switched to CPU EP — DML retry budget exhausted "
|
||||
"or device removed. Restart the process to retry GPU.",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
catch (const std::exception& re) {
|
||||
_logger.LogFatal(context,
|
||||
std::string("DML recreate exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
_dmlRecreateInProgress.store(false);
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXYOLO::DetectObjects(const cv::Mat& inputImage,
|
||||
const std::string& camera_id)
|
||||
{
|
||||
@@ -1985,16 +2141,27 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
|
||||
// ── Snapshot config under a brief lock ──────────────────
|
||||
// ── Snapshot config + engine handle under a brief lock ──
|
||||
// Only hold _mutex long enough to validate state and copy
|
||||
// the parameters needed for inference. The actual ORT
|
||||
// detect() call runs outside the lock so that concurrent
|
||||
// Initialize/LoadModel calls are not blocked for the full
|
||||
// duration of inference.
|
||||
//
|
||||
// B2: We also snapshot a shared_ptr COPY of m_ortEngine
|
||||
// here. The lock-free detect() call below dereferences
|
||||
// the LOCAL copy (`engine`), not the member. If a
|
||||
// concurrent thread catches a DML fatal and does
|
||||
// m_ortEngine.reset() under the same lock, that only
|
||||
// releases the member's reference — our local copy keeps
|
||||
// the engine alive until this function returns. This
|
||||
// closes the use-after-free race that produced the
|
||||
// 0xc0000005 crash at ANSODEngine.dll +0x111fd0.
|
||||
float probThresh, nmsThresh;
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
{
|
||||
ANS_DBG("ONNXYOLO", "DetectObjects: cam=%s acquiring mutex...", camera_id.c_str());
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjects");
|
||||
@@ -2007,13 +2174,14 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Snapshot parameters while locked
|
||||
// Snapshot parameters + engine handle while locked
|
||||
probThresh = PROBABILITY_THRESHOLD;
|
||||
nmsThresh = NMS_THRESHOLD;
|
||||
numKps = NUM_KPS;
|
||||
classes = _classes;
|
||||
trackerEnabled = _trackerEnabled;
|
||||
stabilizationEnabled = _stabilizationEnabled;
|
||||
engine = m_ortEngine; // ref-count++
|
||||
}
|
||||
// ── _mutex released — heavy work below runs lock-free ───
|
||||
|
||||
@@ -2047,11 +2215,28 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Run ORT inference — no mutex held, this is the expensive call
|
||||
auto results = m_ortEngine->detect(inferenceImage, classes,
|
||||
probThresh,
|
||||
nmsThresh,
|
||||
numKps);
|
||||
// Run ORT inference — no engine mutex held, this is the
|
||||
// expensive call.
|
||||
// B2: dereference the LOCAL shared_ptr `engine`, not the
|
||||
// member m_ortEngine. A concurrent recreate thread may
|
||||
// reset() m_ortEngine while we're inside detect() — the
|
||||
// local ref-count keeps the underlying ONNXYOLO alive
|
||||
// until this function returns.
|
||||
//
|
||||
// Option A: DmlInferenceGuard wraps the call to (a)
|
||||
// serialize on the global DML mutex IF DML serial mode has
|
||||
// been tripped, and (b) trip serial mode if THIS call's
|
||||
// latency exceeds the slow threshold (1.5 s — well below
|
||||
// WDDM TDR). No-op on Intel/NVIDIA/CPU.
|
||||
std::vector<Object> results;
|
||||
{
|
||||
DmlInferenceGuard dmlGuard(engine->getEngineType(),
|
||||
"ANSONNXYOLO::DetectObjects");
|
||||
results = engine->detect(inferenceImage, classes,
|
||||
probThresh,
|
||||
nmsThresh,
|
||||
numKps);
|
||||
}
|
||||
|
||||
// --- Rescale coordinates from full-res back to display-res ---
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
@@ -2075,42 +2260,38 @@ namespace ANSCENTER {
|
||||
for (auto& obj : results)
|
||||
obj.cameraId = camera_id;
|
||||
|
||||
// Tracking/stabilization (ApplyTracking has its own lock)
|
||||
if (trackerEnabled && !m_ortEngine->lastWasClassification) {
|
||||
// Tracking/stabilization (ApplyTracking has its own lock).
|
||||
// B2: read lastWasClassification from the LOCAL engine copy
|
||||
// — using m_ortEngine here would re-introduce the race we
|
||||
// just closed for detect().
|
||||
if (trackerEnabled && !engine->lastWasClassification) {
|
||||
results = ApplyTracking(results, camera_id);
|
||||
if (stabilizationEnabled) results = StabilizeDetections(results, camera_id);
|
||||
}
|
||||
|
||||
// A successful inference clears the DML failure budget so
|
||||
// isolated transient hiccups (one E_FAIL every few minutes)
|
||||
// never accumulate toward kMaxConsecutiveDmlFailures and
|
||||
// wrongly demote us to CPU. Only sustained back-to-back
|
||||
// failures with no successful inference between them
|
||||
// exhaust the budget.
|
||||
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
|
||||
return results;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "DML device lost — recreating session on CPU");
|
||||
try {
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
m_ortEngine.reset();
|
||||
if (InitOrtEngine(ANSCENTER::EngineType::CPU)) {
|
||||
_logger.LogInfo("ANSONNXYOLO::DetectObjects",
|
||||
"CPU fallback session created successfully",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "CPU fallback OK");
|
||||
} else {
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
"CPU fallback session creation failed",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
} catch (const std::exception& re) {
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
std::string("CPU fallback exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
// Any DML-fatal HRESULT or DML branded error → rebuild on CPU.
|
||||
// See IsDmlFatalError() in ANSEngineCommon.h for the full set
|
||||
// (887A0005 device-removed, 80004005 generic E_FAIL from
|
||||
// DmlGraphFusionHelper / DmlCommandRecorder, etc.).
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure for full
|
||||
// contract. Tries to rebuild the session on the same
|
||||
// DML EP first; only falls back to CPU on
|
||||
// device-removed (887A0005) or after the retry budget
|
||||
// is exhausted.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::DetectObjects", msg);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -2157,19 +2338,9 @@ namespace ANSCENTER {
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
try {
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
"CPU fallback session creation failed", __FILE__, __LINE__);
|
||||
} catch (...) {}
|
||||
}
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::RunInferencesBatch", msg);
|
||||
return {};
|
||||
}
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
@@ -2184,11 +2355,15 @@ namespace ANSCENTER {
|
||||
try {
|
||||
if (_modelLoading.load()) return {};
|
||||
|
||||
// Snapshot config under brief lock
|
||||
// Snapshot config + engine handle under brief lock.
|
||||
// B2: shared_ptr copy `engine` keeps the underlying ONNXYOLO
|
||||
// alive across the lock-free detectBatch() call even if a
|
||||
// concurrent DML-fatal handler resets m_ortEngine.
|
||||
float probThresh, nmsThresh;
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
{
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjectsBatch");
|
||||
if (!lk.owns_lock()) return {};
|
||||
@@ -2204,13 +2379,20 @@ namespace ANSCENTER {
|
||||
classes = _classes;
|
||||
trackerEnabled = _trackerEnabled;
|
||||
stabilizationEnabled = _stabilizationEnabled;
|
||||
engine = m_ortEngine; // ref-count++
|
||||
}
|
||||
|
||||
// Heavy work outside lock
|
||||
auto batchResults = m_ortEngine->detectBatch(
|
||||
inputImages, classes, probThresh, nmsThresh, numKps);
|
||||
// Heavy work outside lock — dereference local copy, not member.
|
||||
// Option A: same DmlInferenceGuard contract as DetectObjects.
|
||||
std::vector<std::vector<Object>> batchResults;
|
||||
{
|
||||
DmlInferenceGuard dmlGuard(engine->getEngineType(),
|
||||
"ANSONNXYOLO::DetectObjectsBatch");
|
||||
batchResults = engine->detectBatch(
|
||||
inputImages, classes, probThresh, nmsThresh, numKps);
|
||||
}
|
||||
|
||||
const bool isClassification = m_ortEngine->lastBatchWasClassification;
|
||||
const bool isClassification = engine->lastBatchWasClassification;
|
||||
|
||||
for (auto& results : batchResults) {
|
||||
for (auto& obj : results)
|
||||
@@ -2222,24 +2404,16 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Clear DML failure budget on success — see DetectObjects
|
||||
// for the full rationale.
|
||||
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
|
||||
return batchResults;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
try {
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
"CPU fallback session creation failed", __FILE__, __LINE__);
|
||||
} catch (...) {}
|
||||
}
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::DetectObjectsBatch", msg);
|
||||
return {};
|
||||
}
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
|
||||
@@ -210,14 +210,74 @@ namespace ANSCENTER {
|
||||
int NUM_KPS{ 0 };
|
||||
float KPS_THRESHOLD{ 0.5f };
|
||||
|
||||
// ONNX Runtime inference engine
|
||||
std::unique_ptr<ONNXYOLO> m_ortEngine;
|
||||
// ONNX Runtime inference engine.
|
||||
//
|
||||
// B2: shared_ptr (was unique_ptr) so concurrent inference threads
|
||||
// can grab a ref-counted local copy under the brief snapshot lock
|
||||
// and dereference it OUTSIDE the lock without racing with a
|
||||
// concurrent reset() in the DML-fatal CPU-fallback path. Without
|
||||
// this, a thread doing `m_ortEngine->detect(...)` lock-free could
|
||||
// see its underlying object freed mid-call by another thread that
|
||||
// caught a DML E_FAIL and rebuilt the session — that was the
|
||||
// 0xc0000005 access violation observed in ANSODEngine.dll under
|
||||
// AMD/DirectML when multiple AI tasks ran in parallel.
|
||||
std::shared_ptr<ONNXYOLO> m_ortEngine;
|
||||
|
||||
// DML device-lost recovery: when DirectML's GPU device is removed
|
||||
// (HRESULT 887A0005), the session is permanently broken. We detect
|
||||
// this once, attempt a CPU-fallback recreation, and suppress further
|
||||
// error-log flooding.
|
||||
bool _dmlDeviceLost{ false };
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// DML stay-on-GPU recovery state.
|
||||
//
|
||||
// AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from
|
||||
// DmlGraphFusionHelper / DmlCommandRecorder when the driver
|
||||
// chokes under concurrent inference pressure. Empirically this
|
||||
// is NOT a permanent device loss — recreating the ORT session on
|
||||
// the same DML EP clears the wedged state and resumes GPU
|
||||
// inference at full speed. Only the truly-fatal 887A0005
|
||||
// (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback.
|
||||
//
|
||||
// Recovery contract (used by every Run() catch block):
|
||||
// 1. compare_exchange on _dmlRecreateInProgress elects one
|
||||
// thread as the recreate owner. Other threads return {}
|
||||
// and retry on the next call.
|
||||
// 2. The owner checks the cooldown — if we recreated in the
|
||||
// last kDmlRecreateCooldownMs ms, skip (driver is in a
|
||||
// prolonged bad state, don't thrash).
|
||||
// 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit
|
||||
// kMaxConsecutiveDmlFailures recreates without a successful
|
||||
// inference in between, fall back to CPU as last resort.
|
||||
// 4. Otherwise reset() + InitOrtEngine(<original DML EP>) +
|
||||
// WarmUpEngine() to rebuild on the same GPU EP.
|
||||
// 5. On successful inference, _dmlConsecutiveFailures resets
|
||||
// to 0, so isolated transient hiccups don't accumulate
|
||||
// toward the CPU-fallback threshold.
|
||||
//
|
||||
// This replaces the old single-shot `bool _dmlDeviceLost` flag
|
||||
// which latched on the first failure and forced CPU forever —
|
||||
// user requirement is to keep AI tasks running on GPU, not to
|
||||
// trade GPU performance for stability the moment the driver
|
||||
// hiccups once.
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
std::atomic<bool> _dmlRecreateInProgress{ false };
|
||||
std::atomic<int> _dmlConsecutiveFailures{ 0 };
|
||||
std::chrono::steady_clock::time_point _lastDmlRecreateAt{};
|
||||
|
||||
// Tunables — tuned for AMD Radeon 780M observed behavior.
|
||||
// threshold: catches isolated transient AMD DML hiccups. After
|
||||
// 3 consecutive recreates without a successful inference, the
|
||||
// driver is in a prolonged bad state and CPU is the only
|
||||
// forward path.
|
||||
// cooldown: prevents recreate thrash. AMD DML session creation
|
||||
// takes 200–500 ms; if E_FAIL fires from many threads at once
|
||||
// we don't want to recreate N times in a row.
|
||||
static constexpr int kMaxConsecutiveDmlFailures = 3;
|
||||
static constexpr int64_t kDmlRecreateCooldownMs = 5000;
|
||||
|
||||
// Centralised DML-fatal recovery used by all 3 catch sites.
|
||||
// Caller must have already determined the exception is DML-related
|
||||
// (IsDmlFatalError == true). msg is the exception message used
|
||||
// both to log and to detect the device-removed sub-case. Returns
|
||||
// immediately without doing anything when another thread already
|
||||
// owns the recreate (CAS loser path).
|
||||
void HandleDmlRunFailure(const char* context, const std::string& msg);
|
||||
|
||||
// Internal detection pipeline
|
||||
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
|
||||
|
||||
@@ -384,13 +384,17 @@ namespace ANSCENTER {
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
|
||||
// DML device-removal detection (see ANSONNXYOLO.cpp for details)
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
// DML fatal-error detection (887A0005 device-removed, 80004005 generic
|
||||
// DML E_FAIL, branded DmlFusedNode / DmlCommandRecorder / etc.).
|
||||
// See ANSCENTER::IsDmlFatalError in ANSEngineCommon.h.
|
||||
if (IsDmlFatalError(msg)) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSOVFD::RunInference",
|
||||
"DirectML GPU device lost (887A0005) — will attempt CPU fallback on next call",
|
||||
std::string("DirectML fatal error — will attempt CPU fallback on next call. msg=")
|
||||
+ msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSOVFD", "DML fatal msg=%s", msg.c_str());
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user