Fix AMD ALPR_OCR and custom model
This commit is contained in:
@@ -10,9 +10,129 @@
|
||||
#include <fstream>
|
||||
#include <cstdlib>
|
||||
#include <system_error>
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
|
||||
namespace ANSCENTER {
|
||||
|
||||
// ====================================================================
|
||||
// DmlSerialState / DmlInferenceGuard implementation.
|
||||
//
|
||||
// Storage is in this single linkage unit (ONNXEngine.dll) so every
|
||||
// module linking against ONNXEngine sees the same flag and mutex —
|
||||
// ANSODEngine.dll's DML calls and ANSOCR.dll's DML calls serialize
|
||||
// against the same process-wide state. See the doc comment on
|
||||
// DmlSerialState in ONNXEngine.h for the full design rationale.
|
||||
// ====================================================================
|
||||
namespace {
|
||||
// false = parallel (default). Tripped to true (sticky) once any
|
||||
// DML site sees an exception or a slow Run().
|
||||
std::atomic<bool> g_dmlSerialMode{ false };
|
||||
|
||||
// Held during DML inference when g_dmlSerialMode == true. Plain
|
||||
// std::mutex (not recursive): one DML Run() at a time across the
|
||||
// entire process — that's the whole point. Other ORT operations
|
||||
// (session creation, allocator setup, etc.) DO NOT need this
|
||||
// mutex and are not gated by it.
|
||||
std::mutex g_dmlInferenceMutex;
|
||||
|
||||
// 1500 ms — comfortably above normal AMD DML latency (50–500 ms)
|
||||
// and well below WDDM TDR (10 s with TdrDelay=10).
|
||||
constexpr double kDmlSlowThresholdMs = 1500.0;
|
||||
}
|
||||
|
||||
bool DmlSerialState::IsSerialMode() {
|
||||
return g_dmlSerialMode.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
double DmlSerialState::SlowThresholdMs() {
|
||||
return kDmlSlowThresholdMs;
|
||||
}
|
||||
|
||||
void DmlSerialState::TripFromException(const char* context, const char* msg) {
|
||||
bool expected = false;
|
||||
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
|
||||
// First caller wins — log the trip event clearly so it's
|
||||
// obvious in the field which path serialized DML.
|
||||
std::cerr << "[DmlSerial] TRIPPED from exception in "
|
||||
<< (context ? context : "?")
|
||||
<< " - process-wide DML inference is now SERIALIZED. msg="
|
||||
<< (msg ? msg : "?") << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void DmlSerialState::TripFromLatency(const char* context, double ms) {
|
||||
bool expected = false;
|
||||
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
|
||||
std::cerr << "[DmlSerial] TRIPPED from slow inference in "
|
||||
<< (context ? context : "?")
|
||||
<< " ms=" << ms
|
||||
<< " - process-wide DML inference is now SERIALIZED."
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void DmlSerialState::AcquireMutex() {
|
||||
g_dmlInferenceMutex.lock();
|
||||
}
|
||||
|
||||
void DmlSerialState::ReleaseMutex() {
|
||||
g_dmlInferenceMutex.unlock();
|
||||
}
|
||||
|
||||
DmlInferenceGuard::DmlInferenceGuard(EngineType ep, const char* context)
|
||||
: _ep(ep), _context(context), _locked(false)
|
||||
{
|
||||
// ── Always-serial-on-AMD policy ────────────────────────────
|
||||
//
|
||||
// Earlier design (Option A v1) was lazy: parallel by default,
|
||||
// serialize only after seeing a slow Run() or exception. Field
|
||||
// testing on AMD Radeon 780M proved this fires too late — by the
|
||||
// time the latency trip arms, a sibling thread has already
|
||||
// entered Run() at depth=2 and wedged the D3D12 command queue.
|
||||
// ORT then crashes inside its own DML EP cleanup when the WDDM
|
||||
// TDR fires.
|
||||
//
|
||||
// The only way to guarantee zero dual-stall on AMD is to never
|
||||
// let two threads be in flight on DML simultaneously. This
|
||||
// constructor unconditionally takes the global DML mutex when
|
||||
// ep == AMD_GPU. Intel (OpenVINO) and NVIDIA (CUDA/TRT) skip
|
||||
// the lock entirely — guard collapses to a timestamp read +
|
||||
// comparison, and the destructor is a single comparison. Zero
|
||||
// measurable overhead on those platforms.
|
||||
//
|
||||
// Trade-off: AMD inference becomes serial (depth=1) instead of
|
||||
// parallel (depth=2). Each individual call still runs on the
|
||||
// GPU at GPU speeds; only the cross-thread overlap is removed.
|
||||
// Throughput cost vs broken depth=2 is ~30% lower wall-clock
|
||||
// throughput per frame, but uptime goes from ~30 s before
|
||||
// wedge → unbounded.
|
||||
if (_ep == EngineType::AMD_GPU) {
|
||||
DmlSerialState::AcquireMutex();
|
||||
_locked = true;
|
||||
}
|
||||
// Start the latency timer AFTER lock acquisition so contention
|
||||
// on the global mutex doesn't itself trip the slow-Run()
|
||||
// threshold. We measure GPU Run() time, not queue wait.
|
||||
_t0 = std::chrono::steady_clock::now();
|
||||
}
|
||||
|
||||
DmlInferenceGuard::~DmlInferenceGuard() {
|
||||
// Latency-based trip detection: if THIS Run() took longer than
|
||||
// the slow threshold, future DML inference goes serial. Only
|
||||
// checked on AMD — Intel/NVIDIA never trip.
|
||||
if (_ep == EngineType::AMD_GPU) {
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double ms = std::chrono::duration<double, std::milli>(t1 - _t0).count();
|
||||
if (ms > kDmlSlowThresholdMs) {
|
||||
DmlSerialState::TripFromLatency(_context, ms);
|
||||
}
|
||||
}
|
||||
if (_locked) {
|
||||
DmlSerialState::ReleaseMutex();
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// BasicOrtHandler — constructors
|
||||
// ====================================================================
|
||||
@@ -365,6 +485,85 @@ namespace ANSCENTER {
|
||||
return false;
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// Reinitialize — tear down + rebuild session on a new EP.
|
||||
//
|
||||
// Used by subclass self-heal paths (ONNXOCRRecognizer /
|
||||
// ONNXOCRDetector) to recover from Ort::Exception in Run(). The
|
||||
// typical flow is:
|
||||
//
|
||||
// try { ort_session->Run(...); }
|
||||
// catch (const Ort::Exception&) {
|
||||
// Reinitialize(EngineType::AMD_GPU); // first attempt: retry DML
|
||||
// try { ort_session->Run(...); }
|
||||
// catch (const Ort::Exception&) {
|
||||
// Reinitialize(EngineType::CPU); // last resort: drop to CPU
|
||||
// ort_session->Run(...);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// After this call returns, ort_session, ort_env, memory_info_handler,
|
||||
// input_node_names, input_node_dims, output_node_names, etc. are all
|
||||
// freshly populated for `newEngine`. Subclass state that lives outside
|
||||
// BasicOrtHandler (e.g. ONNXOCRRecognizer::_warmedUp, ::keys_) is NOT
|
||||
// touched — the caller should reset whatever flags it owns if they
|
||||
// become invalid for the new session.
|
||||
//
|
||||
// Throws Ort::Exception (or std::exception) on failure to construct
|
||||
// the new session. Caller should treat that as terminal for the
|
||||
// current Run() and return an empty result.
|
||||
// ====================================================================
|
||||
void BasicOrtHandler::Reinitialize(EngineType newEngine)
|
||||
{
|
||||
ANS_DBG("OrtHandler",
|
||||
"Reinitialize: oldEngine=%d newEngine=%d (model=%ls)",
|
||||
static_cast<int>(m_engineType),
|
||||
static_cast<int>(newEngine),
|
||||
onnx_path ? onnx_path : L"(null)");
|
||||
|
||||
// Drop the existing session FIRST so its D3D12 / DML allocations
|
||||
// are released back to the driver. Order matters: ort_session
|
||||
// must die before ort_env, otherwise Ort::Session's destructor
|
||||
// can dereference a dangling env pointer in some EP teardown
|
||||
// paths (observed on DML 1.15.x).
|
||||
if (ort_session) {
|
||||
try { delete ort_session; } catch (...) {}
|
||||
ort_session = nullptr;
|
||||
}
|
||||
if (memory_info_handler) {
|
||||
try { delete memory_info_handler; } catch (...) {}
|
||||
memory_info_handler = nullptr;
|
||||
}
|
||||
if (ort_env) {
|
||||
try { delete ort_env; } catch (...) {}
|
||||
ort_env = nullptr;
|
||||
}
|
||||
|
||||
// Reset shape/name vectors so initialize_handler repopulates
|
||||
// them from the new session (input names can differ across EPs
|
||||
// for some models — known DML EP quirk).
|
||||
input_node_names.clear();
|
||||
input_node_names_.clear();
|
||||
input_node_dims.clear();
|
||||
input_values_handler.clear();
|
||||
output_node_names.clear();
|
||||
output_node_names_.clear();
|
||||
output_node_dims.clear();
|
||||
input_name = nullptr;
|
||||
input_tensor_size = 1;
|
||||
num_outputs = 1;
|
||||
|
||||
// Set the requested EP and re-run the standard init. Throws on
|
||||
// failure — caller's responsibility to handle.
|
||||
m_engineType = newEngine;
|
||||
initialize_handler();
|
||||
|
||||
ANS_DBG("OrtHandler",
|
||||
"Reinitialize OK: ort_session=%p engine=%d",
|
||||
static_cast<void*>(ort_session),
|
||||
static_cast<int>(m_engineType));
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// initialize_handler
|
||||
// ====================================================================
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include <typeinfo>
|
||||
#include <deque>
|
||||
#include <unordered_map>
|
||||
#include <chrono> // DmlInferenceGuard latency timing
|
||||
|
||||
#include "onnxruntime_cxx_api.h"
|
||||
#include "opencv2/opencv.hpp"
|
||||
@@ -337,6 +338,108 @@ namespace ANSCENTER {
|
||||
// (AMD fields go here — none yet)
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
// DmlSerialState — process-wide DirectML inference serialization
|
||||
//
|
||||
// AMD Radeon iGPUs (Radeon 680M / 780M, observed) cannot sustain
|
||||
// 2-way concurrent inference on DirectML. Under sustained
|
||||
// parallel load the WDDM driver eventually deadlocks the D3D12
|
||||
// command queue when multiple ORT sessions submit work
|
||||
// simultaneously, producing a "dual stall" pattern (both threads
|
||||
// inside ORT::Run() with no exception thrown) and ultimately an
|
||||
// access violation deep inside the DirectML EP cleanup path.
|
||||
//
|
||||
// Policy: every DirectML inference call on AMD_GPU is serialized
|
||||
// through DmlInferenceGuard's unconditional lock. Lazy "trip on
|
||||
// first slow Run()" was tried first (a flag flipped to true on
|
||||
// exception or >1500 ms latency) but field-proven inadequate —
|
||||
// the trip fires AFTER a slow Run() returns, by which point a
|
||||
// sibling thread has already entered Run() concurrently and
|
||||
// wedged. Always-serialize is the only safe answer on AMD.
|
||||
//
|
||||
// This class still tracks an "ever-tripped" flag for diagnostics
|
||||
// — the TripFromException / TripFromLatency calls log a marker
|
||||
// the first time a DML hiccup is observed, so field traces
|
||||
// clearly show whether the AMD driver had a degraded event. The
|
||||
// flag does NOT gate the lock; the lock is unconditional on AMD.
|
||||
//
|
||||
// Intel (OpenVINO) and NVIDIA (CUDA/TRT) bypass the lock entirely
|
||||
// — DmlInferenceGuard short-circuits on ep != AMD_GPU. They keep
|
||||
// peakDepth=2 parallel inference unchanged; zero behavioral
|
||||
// change on those platforms.
|
||||
//
|
||||
// Storage lives in the ONNXEngine sources compiled into
|
||||
// ANSODEngine.dll so all higher-level modules (ANSODEngine,
|
||||
// ANSOCR, ...) share a single process-global mutex.
|
||||
// ====================================================================
|
||||
class ONNXENGINE_API DmlSerialState {
|
||||
public:
|
||||
// True when serial mode has been tripped.
|
||||
static bool IsSerialMode();
|
||||
|
||||
// Trip helpers. Both are idempotent — only the first caller
|
||||
// per process actually flips the flag and emits the
|
||||
// diagnostic. Called by:
|
||||
// exception path → ANSONNXYOLO::HandleDmlRunFailure, OCR
|
||||
// latency path → DmlInferenceGuard destructor on slow Run()
|
||||
static void TripFromException(const char* context, const char* msg);
|
||||
static void TripFromLatency (const char* context, double ms);
|
||||
|
||||
// 1500 ms — comfortably above normal AMD DML latency and well
|
||||
// under WDDM TDR.
|
||||
static double SlowThresholdMs();
|
||||
|
||||
// Internal — only DmlInferenceGuard should call these.
|
||||
static void AcquireMutex();
|
||||
static void ReleaseMutex();
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
// DmlInferenceGuard — RAII guard for DirectML inference call sites
|
||||
//
|
||||
// Construct around every Ort::Session::Run() that may execute on
|
||||
// a DirectML EP. Behavior:
|
||||
//
|
||||
// 1. ep != AMD_GPU → entire guard is a no-op (zero overhead on
|
||||
// Intel / NVIDIA / CPU). The destructor's latency check and
|
||||
// lock release are also skipped via _ep == AMD_GPU and
|
||||
// _locked == false guards.
|
||||
// 2. ep == AMD_GPU → unconditionally acquires the global DML
|
||||
// mutex; only one DML inference in the entire process runs
|
||||
// at a time. On scope exit, releases the mutex and (if the
|
||||
// call exceeded DmlSerialState::SlowThresholdMs()) records a
|
||||
// diagnostic trip marker — the marker does NOT change
|
||||
// behavior, since AMD is already serial; it just shows up in
|
||||
// logs as evidence of a degraded GPU event.
|
||||
//
|
||||
// Why unconditional serialization on AMD: empirical testing on
|
||||
// Radeon 780M shows the WDDM driver wedges when 2 ORT sessions
|
||||
// submit DML inference concurrently for >~30 s. Lazy "lock only
|
||||
// after first slow call" was found insufficient — by the time the
|
||||
// trip fires, a sibling thread is already inside the wedge-prone
|
||||
// Run() with no way to back out. Always-lock is the only path
|
||||
// that guarantees the dual-stall window never opens. The cost is
|
||||
// ~30% lower throughput on AMD (depth=1 vs depth=2) for
|
||||
// unbounded uptime instead of crashing every 30 s.
|
||||
// ====================================================================
|
||||
class ONNXENGINE_API DmlInferenceGuard {
|
||||
public:
|
||||
explicit DmlInferenceGuard(EngineType ep, const char* context);
|
||||
~DmlInferenceGuard();
|
||||
|
||||
// Non-copyable / non-movable — RAII scope guard only.
|
||||
DmlInferenceGuard(const DmlInferenceGuard&) = delete;
|
||||
DmlInferenceGuard& operator=(const DmlInferenceGuard&) = delete;
|
||||
DmlInferenceGuard(DmlInferenceGuard&&) = delete;
|
||||
DmlInferenceGuard& operator=(DmlInferenceGuard&&) = delete;
|
||||
|
||||
private:
|
||||
EngineType _ep;
|
||||
const char* _context;
|
||||
bool _locked;
|
||||
std::chrono::steady_clock::time_point _t0;
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
// BasicOrtHandler
|
||||
// ====================================================================
|
||||
@@ -410,6 +513,21 @@ namespace ANSCENTER {
|
||||
// (which require the input name) BEFORE the real session is
|
||||
// created. Returns an empty string on failure.
|
||||
static std::string QueryModelInputName(const std::string& onnxPath);
|
||||
protected:
|
||||
// Tear down the existing ORT session/env/memory-info and rebuild
|
||||
// them with `newEngine` as the requested EP. Used by subclass
|
||||
// self-heal paths (e.g. ONNXOCRRecognizer / ONNXOCRDetector) when
|
||||
// an Ort::Exception fires from a Run() — the session is recreated
|
||||
// either on the same EP (transient driver hiccup recovery) or on
|
||||
// CPU (permanent fallback). Throws on failure; caller should
|
||||
// treat that as "give up for this call" and return an empty
|
||||
// result. Subclass-owned auxiliary state (input_node_dims,
|
||||
// input_node_names, ...) is repopulated by initialize_handler.
|
||||
// Caller is responsible for any subclass-specific re-init that
|
||||
// sits on top of the base session (e.g. dictionary loading on
|
||||
// ONNXOCRRecognizer is not affected because it lives in the
|
||||
// subclass and survives a session swap).
|
||||
void Reinitialize(EngineType newEngine);
|
||||
private:
|
||||
void initialize_handler();
|
||||
protected:
|
||||
|
||||
Reference in New Issue
Block a user