Fix AMD ALPR_OCR and custom model

This commit is contained in:
2026-05-01 07:53:18 +10:00
parent e1f37ec767
commit 91efbb9a8f
25 changed files with 2841 additions and 429 deletions
+199
View File
@@ -10,9 +10,129 @@
#include <fstream>
#include <cstdlib>
#include <system_error>
#include <atomic>
#include <mutex>
namespace ANSCENTER {
// ====================================================================
// DmlSerialState / DmlInferenceGuard implementation.
//
// Storage is in this single linkage unit (ONNXEngine.dll) so every
// module linking against ONNXEngine sees the same flag and mutex —
// ANSODEngine.dll's DML calls and ANSOCR.dll's DML calls serialize
// against the same process-wide state. See the doc comment on
// DmlSerialState in ONNXEngine.h for the full design rationale.
// ====================================================================
namespace {
// false = parallel (default). Tripped to true (sticky) once any
// DML site sees an exception or a slow Run().
std::atomic<bool> g_dmlSerialMode{ false };
// Held during DML inference when g_dmlSerialMode == true. Plain
// std::mutex (not recursive): one DML Run() at a time across the
// entire process — that's the whole point. Other ORT operations
// (session creation, allocator setup, etc.) DO NOT need this
// mutex and are not gated by it.
std::mutex g_dmlInferenceMutex;
// 1500 ms — comfortably above normal AMD DML latency (50–500 ms)
// and well below WDDM TDR (10 s with TdrDelay=10).
constexpr double kDmlSlowThresholdMs = 1500.0;
}
bool DmlSerialState::IsSerialMode() {
return g_dmlSerialMode.load(std::memory_order_relaxed);
}
double DmlSerialState::SlowThresholdMs() {
return kDmlSlowThresholdMs;
}
void DmlSerialState::TripFromException(const char* context, const char* msg) {
bool expected = false;
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
// First caller wins — log the trip event clearly so it's
// obvious in the field which path serialized DML.
std::cerr << "[DmlSerial] TRIPPED from exception in "
<< (context ? context : "?")
<< " - process-wide DML inference is now SERIALIZED. msg="
<< (msg ? msg : "?") << std::endl;
}
}
void DmlSerialState::TripFromLatency(const char* context, double ms) {
bool expected = false;
if (g_dmlSerialMode.compare_exchange_strong(expected, true)) {
std::cerr << "[DmlSerial] TRIPPED from slow inference in "
<< (context ? context : "?")
<< " ms=" << ms
<< " - process-wide DML inference is now SERIALIZED."
<< std::endl;
}
}
void DmlSerialState::AcquireMutex() {
g_dmlInferenceMutex.lock();
}
void DmlSerialState::ReleaseMutex() {
g_dmlInferenceMutex.unlock();
}
DmlInferenceGuard::DmlInferenceGuard(EngineType ep, const char* context)
: _ep(ep), _context(context), _locked(false)
{
// ── Always-serial-on-AMD policy ────────────────────────────
//
// Earlier design (Option A v1) was lazy: parallel by default,
// serialize only after seeing a slow Run() or exception. Field
// testing on AMD Radeon 780M proved this fires too late — by the
// time the latency trip arms, a sibling thread has already
// entered Run() at depth=2 and wedged the D3D12 command queue.
// ORT then crashes inside its own DML EP cleanup when the WDDM
// TDR fires.
//
// The only way to guarantee zero dual-stall on AMD is to never
// let two threads be in flight on DML simultaneously. This
// constructor unconditionally takes the global DML mutex when
// ep == AMD_GPU. Intel (OpenVINO) and NVIDIA (CUDA/TRT) skip
// the lock entirely — guard collapses to a timestamp read +
// comparison, and the destructor is a single comparison. Zero
// measurable overhead on those platforms.
//
// Trade-off: AMD inference becomes serial (depth=1) instead of
// parallel (depth=2). Each individual call still runs on the
// GPU at GPU speeds; only the cross-thread overlap is removed.
// Throughput cost vs broken depth=2 is ~30% lower wall-clock
// throughput per frame, but uptime goes from ~30 s before
// wedge → unbounded.
if (_ep == EngineType::AMD_GPU) {
DmlSerialState::AcquireMutex();
_locked = true;
}
// Start the latency timer AFTER lock acquisition so contention
// on the global mutex doesn't itself trip the slow-Run()
// threshold. We measure GPU Run() time, not queue wait.
_t0 = std::chrono::steady_clock::now();
}
DmlInferenceGuard::~DmlInferenceGuard() {
// Latency-based trip detection: if THIS Run() took longer than
// the slow threshold, future DML inference goes serial. Only
// checked on AMD — Intel/NVIDIA never trip.
if (_ep == EngineType::AMD_GPU) {
const auto t1 = std::chrono::steady_clock::now();
const double ms = std::chrono::duration<double, std::milli>(t1 - _t0).count();
if (ms > kDmlSlowThresholdMs) {
DmlSerialState::TripFromLatency(_context, ms);
}
}
if (_locked) {
DmlSerialState::ReleaseMutex();
}
}
// ====================================================================
// BasicOrtHandler — constructors
// ====================================================================
@@ -365,6 +485,85 @@ namespace ANSCENTER {
return false;
}
// ====================================================================
// Reinitialize — tear down + rebuild session on a new EP.
//
// Used by subclass self-heal paths (ONNXOCRRecognizer /
// ONNXOCRDetector) to recover from Ort::Exception in Run(). The
// typical flow is:
//
// try { ort_session->Run(...); }
// catch (const Ort::Exception&) {
// Reinitialize(EngineType::AMD_GPU); // first attempt: retry DML
// try { ort_session->Run(...); }
// catch (const Ort::Exception&) {
// Reinitialize(EngineType::CPU); // last resort: drop to CPU
// ort_session->Run(...);
// }
// }
//
// After this call returns, ort_session, ort_env, memory_info_handler,
// input_node_names, input_node_dims, output_node_names, etc. are all
// freshly populated for `newEngine`. Subclass state that lives outside
// BasicOrtHandler (e.g. ONNXOCRRecognizer::_warmedUp, ::keys_) is NOT
// touched — the caller should reset whatever flags it owns if they
// become invalid for the new session.
//
// Throws Ort::Exception (or std::exception) on failure to construct
// the new session. Caller should treat that as terminal for the
// current Run() and return an empty result.
// ====================================================================
void BasicOrtHandler::Reinitialize(EngineType newEngine)
{
ANS_DBG("OrtHandler",
"Reinitialize: oldEngine=%d newEngine=%d (model=%ls)",
static_cast<int>(m_engineType),
static_cast<int>(newEngine),
onnx_path ? onnx_path : L"(null)");
// Drop the existing session FIRST so its D3D12 / DML allocations
// are released back to the driver. Order matters: ort_session
// must die before ort_env, otherwise Ort::Session's destructor
// can dereference a dangling env pointer in some EP teardown
// paths (observed on DML 1.15.x).
if (ort_session) {
try { delete ort_session; } catch (...) {}
ort_session = nullptr;
}
if (memory_info_handler) {
try { delete memory_info_handler; } catch (...) {}
memory_info_handler = nullptr;
}
if (ort_env) {
try { delete ort_env; } catch (...) {}
ort_env = nullptr;
}
// Reset shape/name vectors so initialize_handler repopulates
// them from the new session (input names can differ across EPs
// for some models — known DML EP quirk).
input_node_names.clear();
input_node_names_.clear();
input_node_dims.clear();
input_values_handler.clear();
output_node_names.clear();
output_node_names_.clear();
output_node_dims.clear();
input_name = nullptr;
input_tensor_size = 1;
num_outputs = 1;
// Set the requested EP and re-run the standard init. Throws on
// failure — caller's responsibility to handle.
m_engineType = newEngine;
initialize_handler();
ANS_DBG("OrtHandler",
"Reinitialize OK: ort_session=%p engine=%d",
static_cast<void*>(ort_session),
static_cast<int>(m_engineType));
}
// ====================================================================
// initialize_handler
// ====================================================================