Fix AMD ALPR_OCR and custom model
This commit is contained in:
@@ -1610,7 +1610,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath);
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1626,7 +1626,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath, engineType);
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath, engineType);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1638,26 +1638,35 @@ namespace ANSCENTER {
|
||||
// ========================================================================
|
||||
// WarmUpEngine — run a dummy inference after session creation.
|
||||
//
|
||||
// Scope: **NVIDIA (CUDA EP) only.** On first inference, the CUDA EP
|
||||
// allocates its memory arena (capped at 2 GB via BasicOrtHandler config),
|
||||
// resolves cuDNN convolution algorithms, and populates the kernel launch
|
||||
// cache. Running one dummy inference at load time amortises this cost
|
||||
// so the first real frame doesn't see a latency spike.
|
||||
// Scope: NVIDIA (CUDA EP) AND AMD (DirectML EP).
|
||||
// • NVIDIA: amortises CUDA arena allocation, cuDNN convolution algo
|
||||
// selection, and kernel-launch cache population.
|
||||
// • AMD: amortises D3D12 PSO compilation and first-time DML kernel
|
||||
// binding. Empirically ~150 ms on DirectML (see comment in
|
||||
// ONNXOCRRecognizer::Warmup). Without this, that compile cost
|
||||
// surfaces inside the first live Run() — which on AMD has been
|
||||
// observed to race with concurrent DML traffic from other sessions
|
||||
// (e.g. helmet detector running while ALPR's _lpDetector fires its
|
||||
// first ever Run()) and trigger DmlGraphFusionHelper crashes /
|
||||
// 887A0006 DEVICE_HUNG. Moving the heavy first-run work into
|
||||
// LoadEngine — where only the loading thread is active — sidesteps
|
||||
// this timing window.
|
||||
//
|
||||
// Explicitly disabled on AMD, Intel and CPU:
|
||||
// • AMD (DirectML) — calling detect() at load time has been observed
|
||||
// to hit a multi-terabyte cv::Mat allocation inside postprocessLegacy
|
||||
// on AMD RDNA iGPUs when DirectML returns garbage output tensor
|
||||
// dims. ONNXYOLO::detect() now has an output-shape sanity guard
|
||||
// that catches this at runtime, so the warm-up would add risk
|
||||
// without benefit. Earlier builds enabled warm-up specifically for
|
||||
// Radeon 680M TDR mitigation; that workaround is obsolete with
|
||||
// current DirectML 1.15.x drivers.
|
||||
// • Intel (OpenVINO) — running detect() at load time has been
|
||||
// observed to expose latent heap-corruption bugs
|
||||
// (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
|
||||
// • CPU EP — no shader compile or kernel cache to warm up; the first
|
||||
// real frame has the same latency as any subsequent frame.
|
||||
// Skipped for:
|
||||
// • Intel (OpenVINO EP) — OpenVINO compiles the full plan at session
|
||||
// creation, so first Run() has no extra compile cost. Calling
|
||||
// detect() at load time has also exposed latent heap-corruption
|
||||
// bugs (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
|
||||
// • CPU EP — no shader / kernel cache to warm up.
|
||||
//
|
||||
// History note: AMD warm-up was disabled in earlier builds after
|
||||
// observing multi-terabyte cv::Mat allocations from garbage DirectML
|
||||
// output dims. ONNXYOLO::detect() now has an output-shape sanity
|
||||
// guard (see line ~257, kMaxOutputDim = 1,000,000) that catches that
|
||||
// case before postprocessLegacy can allocate, so re-enabling AMD
|
||||
// warm-up is safe. The exception-swallowing catch blocks below
|
||||
// guarantee that any warm-up failure is non-fatal — the session is
|
||||
// already loaded and real inference will still work.
|
||||
//
|
||||
// Non-fatal on failure: if warm-up itself throws, regular inference
|
||||
// still works — the engine is fully loaded before WarmUpEngine runs.
|
||||
@@ -1665,9 +1674,11 @@ namespace ANSCENTER {
|
||||
void ANSONNXYOLO::WarmUpEngine() {
|
||||
if (!m_ortEngine) return;
|
||||
|
||||
// Gate strictly on NVIDIA_GPU. Every other EP is a no-op.
|
||||
if (m_ortEngine->getEngineType() != EngineType::NVIDIA_GPU) {
|
||||
ANS_DBG("ONNXYOLO", "Warm-up skipped (non-NVIDIA EP)");
|
||||
// Warm up on NVIDIA and AMD (both have lazy first-Run() compile cost).
|
||||
// Skip on CPU and OpenVINO — see scope comment above.
|
||||
const auto et = m_ortEngine->getEngineType();
|
||||
if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) {
|
||||
ANS_DBG("ONNXYOLO", "Warm-up skipped (CPU/OpenVINO EP)");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1696,7 +1707,8 @@ namespace ANSCENTER {
|
||||
// preprocessing (114,114,114 ~ 128) and avoids degenerate inputs.
|
||||
cv::Mat dummy(h, w, CV_8UC3, cv::Scalar(128, 128, 128));
|
||||
|
||||
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy CUDA inference (%dx%d)", w, h);
|
||||
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy %s inference (%dx%d)",
|
||||
(et == EngineType::NVIDIA_GPU) ? "CUDA" : "DirectML", w, h);
|
||||
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
(void)m_ortEngine->detect(dummy, _classes,
|
||||
@@ -1975,6 +1987,150 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// HandleDmlRunFailure — stay-on-DML recovery (replaces CPU-fallback).
|
||||
//
|
||||
// Called from every catch block that wraps an Ort::Session::Run on a
|
||||
// DirectML EP. The contract is documented on the member declaration
|
||||
// in ANSONNXYOLO.h — quick summary:
|
||||
//
|
||||
// * CAS on _dmlRecreateInProgress elects ONE recreate owner per
|
||||
// driver hiccup; other concurrent failures just bail.
|
||||
// * Cooldown (kDmlRecreateCooldownMs) prevents thrashing if E_FAIL
|
||||
// keeps firing back to back.
|
||||
// * IsDmlDeviceRemoved (887A0005) → CPU fallback, last resort.
|
||||
// * After kMaxConsecutiveDmlFailures recreates without an
|
||||
// intervening successful inference → CPU fallback.
|
||||
// * Otherwise: reset() + InitOrtEngine(<original DML EP>) +
|
||||
// WarmUpEngine() → fresh DML session, GPU inference resumes.
|
||||
//
|
||||
// Successful inferences reset _dmlConsecutiveFailures to 0 in
|
||||
// DetectObjects / DetectObjectsBatch, so isolated transient hiccups
|
||||
// do not accumulate toward the CPU-fallback threshold.
|
||||
// ====================================================================
|
||||
void ANSONNXYOLO::HandleDmlRunFailure(const char* context, const std::string& msg)
|
||||
{
|
||||
// Option A: a DML exception is the strongest possible signal
|
||||
// that the AMD driver can't sustain parallel inference. Trip
|
||||
// process-wide DML serial mode immediately so the NEXT
|
||||
// inference attempt (from any thread, in this DLL or another)
|
||||
// takes the global mutex and runs one-at-a-time. This is
|
||||
// idempotent — only the first caller per process actually
|
||||
// flips the flag. Done before the CAS so even the CAS losers
|
||||
// (concurrent threads catching the same fatal) contribute the
|
||||
// signal — but in practice the first one wins.
|
||||
ANSCENTER::DmlSerialState::TripFromException(context, msg.c_str());
|
||||
|
||||
// CAS — only one thread owns the recreate at a time. Other
|
||||
// concurrent DML failures just return and retry on the next call.
|
||||
bool expected = false;
|
||||
if (!_dmlRecreateInProgress.compare_exchange_strong(expected, true)) {
|
||||
ANS_DBG("ONNXYOLO", "%s: DML fatal — another thread is recreating, skip",
|
||||
context);
|
||||
return;
|
||||
}
|
||||
|
||||
// Hold the engine mutex for the entire recreate so concurrent
|
||||
// inference threads (which use a shared_ptr local copy of the
|
||||
// engine) don't see a half-built session.
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
|
||||
try {
|
||||
// Cooldown — if we just recreated within the window, the
|
||||
// driver is in a prolonged bad state; don't burn time
|
||||
// recreating again immediately. Other threads will keep
|
||||
// returning {} until either the next call past cooldown
|
||||
// succeeds or we hit the CPU-fallback threshold.
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLastMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastDmlRecreateAt).count();
|
||||
if (_lastDmlRecreateAt.time_since_epoch().count() != 0
|
||||
&& sinceLastMs < kDmlRecreateCooldownMs) {
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"%s: DML fatal — within cooldown (%lld ms < %lld), skip",
|
||||
context, (long long)sinceLastMs, (long long)kDmlRecreateCooldownMs);
|
||||
_dmlRecreateInProgress.store(false);
|
||||
return;
|
||||
}
|
||||
_lastDmlRecreateAt = now;
|
||||
|
||||
// Decide target EP for the new session:
|
||||
// * 887A0005 device-removed → CPU is the only option
|
||||
// * Too many consecutive failures → CPU as last resort
|
||||
// * Otherwise → recreate on the SAME DML EP we were on
|
||||
ANSCENTER::EngineType originalEp = ANSCENTER::EngineType::CPU;
|
||||
if (m_ortEngine) {
|
||||
originalEp = m_ortEngine->getEngineType();
|
||||
}
|
||||
|
||||
const int failures = _dmlConsecutiveFailures.fetch_add(1) + 1;
|
||||
const bool deviceRemoved = ANSCENTER::IsDmlDeviceRemoved(msg);
|
||||
const bool exceededRetries = failures > kMaxConsecutiveDmlFailures;
|
||||
|
||||
ANSCENTER::EngineType targetEp = originalEp;
|
||||
const char* reason = "stay-on-DML retry";
|
||||
if (deviceRemoved) {
|
||||
targetEp = ANSCENTER::EngineType::CPU;
|
||||
reason = "device-removed (887A0005) → CPU fallback";
|
||||
} else if (exceededRetries) {
|
||||
targetEp = ANSCENTER::EngineType::CPU;
|
||||
reason = "exceeded retry budget → CPU fallback";
|
||||
}
|
||||
|
||||
_logger.LogFatal(context,
|
||||
std::string("DirectML fatal — ") + reason
|
||||
+ " (failure #" + std::to_string(failures)
|
||||
+ "). msg=" + msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"%s: DML fatal recreate — %s, failures=%d, originalEp=%d, targetEp=%d",
|
||||
context, reason, failures, (int)originalEp, (int)targetEp);
|
||||
|
||||
// reset() releases the member's reference; concurrent inference
|
||||
// threads holding a shared_ptr local copy keep their old
|
||||
// engine alive until they return — no use-after-free (B2).
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(targetEp)) {
|
||||
_logger.LogFatal(context,
|
||||
"DML recreate failed — m_ortEngine left null", __FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "%s: InitOrtEngine(%d) failed", context, (int)targetEp);
|
||||
_dmlRecreateInProgress.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Warm up the new session so the first real call doesn't pay
|
||||
// the algorithm-selection / shader-compile tax. AMD DML
|
||||
// recreate WITHOUT warmup typically hits 1-2 s on the very
|
||||
// first inference, which can re-trigger our 2 s slow watchdog
|
||||
// and produce a false alarm.
|
||||
try {
|
||||
WarmUpEngine();
|
||||
} catch (const std::exception& we) {
|
||||
ANS_DBG("ONNXYOLO", "%s: WarmUpEngine after recreate threw: %s",
|
||||
context, we.what());
|
||||
}
|
||||
|
||||
// If we landed on CPU as a last resort, leave the failure
|
||||
// counter pinned so we don't oscillate back to DML on the
|
||||
// next exception. Otherwise (we're back on DML) DON'T reset
|
||||
// the counter here — it gets reset by a *successful*
|
||||
// inference in DetectObjects / DetectObjectsBatch.
|
||||
if (targetEp == ANSCENTER::EngineType::CPU
|
||||
&& originalEp != ANSCENTER::EngineType::CPU) {
|
||||
_logger.LogInfo(context,
|
||||
"Permanently switched to CPU EP — DML retry budget exhausted "
|
||||
"or device removed. Restart the process to retry GPU.",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
catch (const std::exception& re) {
|
||||
_logger.LogFatal(context,
|
||||
std::string("DML recreate exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
_dmlRecreateInProgress.store(false);
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXYOLO::DetectObjects(const cv::Mat& inputImage,
|
||||
const std::string& camera_id)
|
||||
{
|
||||
@@ -1985,16 +2141,27 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
|
||||
// ── Snapshot config under a brief lock ──────────────────
|
||||
// ── Snapshot config + engine handle under a brief lock ──
|
||||
// Only hold _mutex long enough to validate state and copy
|
||||
// the parameters needed for inference. The actual ORT
|
||||
// detect() call runs outside the lock so that concurrent
|
||||
// Initialize/LoadModel calls are not blocked for the full
|
||||
// duration of inference.
|
||||
//
|
||||
// B2: We also snapshot a shared_ptr COPY of m_ortEngine
|
||||
// here. The lock-free detect() call below dereferences
|
||||
// the LOCAL copy (`engine`), not the member. If a
|
||||
// concurrent thread catches a DML fatal and does
|
||||
// m_ortEngine.reset() under the same lock, that only
|
||||
// releases the member's reference — our local copy keeps
|
||||
// the engine alive until this function returns. This
|
||||
// closes the use-after-free race that produced the
|
||||
// 0xc0000005 crash at ANSODEngine.dll +0x111fd0.
|
||||
float probThresh, nmsThresh;
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
{
|
||||
ANS_DBG("ONNXYOLO", "DetectObjects: cam=%s acquiring mutex...", camera_id.c_str());
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjects");
|
||||
@@ -2007,13 +2174,14 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Snapshot parameters while locked
|
||||
// Snapshot parameters + engine handle while locked
|
||||
probThresh = PROBABILITY_THRESHOLD;
|
||||
nmsThresh = NMS_THRESHOLD;
|
||||
numKps = NUM_KPS;
|
||||
classes = _classes;
|
||||
trackerEnabled = _trackerEnabled;
|
||||
stabilizationEnabled = _stabilizationEnabled;
|
||||
engine = m_ortEngine; // ref-count++
|
||||
}
|
||||
// ── _mutex released — heavy work below runs lock-free ───
|
||||
|
||||
@@ -2047,11 +2215,28 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Run ORT inference — no mutex held, this is the expensive call
|
||||
auto results = m_ortEngine->detect(inferenceImage, classes,
|
||||
probThresh,
|
||||
nmsThresh,
|
||||
numKps);
|
||||
// Run ORT inference — no engine mutex held, this is the
|
||||
// expensive call.
|
||||
// B2: dereference the LOCAL shared_ptr `engine`, not the
|
||||
// member m_ortEngine. A concurrent recreate thread may
|
||||
// reset() m_ortEngine while we're inside detect() — the
|
||||
// local ref-count keeps the underlying ONNXYOLO alive
|
||||
// until this function returns.
|
||||
//
|
||||
// Option A: DmlInferenceGuard wraps the call to (a)
|
||||
// serialize on the global DML mutex IF DML serial mode has
|
||||
// been tripped, and (b) trip serial mode if THIS call's
|
||||
// latency exceeds the slow threshold (1.5 s — well below
|
||||
// WDDM TDR). No-op on Intel/NVIDIA/CPU.
|
||||
std::vector<Object> results;
|
||||
{
|
||||
DmlInferenceGuard dmlGuard(engine->getEngineType(),
|
||||
"ANSONNXYOLO::DetectObjects");
|
||||
results = engine->detect(inferenceImage, classes,
|
||||
probThresh,
|
||||
nmsThresh,
|
||||
numKps);
|
||||
}
|
||||
|
||||
// --- Rescale coordinates from full-res back to display-res ---
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
@@ -2075,42 +2260,38 @@ namespace ANSCENTER {
|
||||
for (auto& obj : results)
|
||||
obj.cameraId = camera_id;
|
||||
|
||||
// Tracking/stabilization (ApplyTracking has its own lock)
|
||||
if (trackerEnabled && !m_ortEngine->lastWasClassification) {
|
||||
// Tracking/stabilization (ApplyTracking has its own lock).
|
||||
// B2: read lastWasClassification from the LOCAL engine copy
|
||||
// — using m_ortEngine here would re-introduce the race we
|
||||
// just closed for detect().
|
||||
if (trackerEnabled && !engine->lastWasClassification) {
|
||||
results = ApplyTracking(results, camera_id);
|
||||
if (stabilizationEnabled) results = StabilizeDetections(results, camera_id);
|
||||
}
|
||||
|
||||
// A successful inference clears the DML failure budget so
|
||||
// isolated transient hiccups (one E_FAIL every few minutes)
|
||||
// never accumulate toward kMaxConsecutiveDmlFailures and
|
||||
// wrongly demote us to CPU. Only sustained back-to-back
|
||||
// failures with no successful inference between them
|
||||
// exhaust the budget.
|
||||
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
|
||||
return results;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "DML device lost — recreating session on CPU");
|
||||
try {
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
m_ortEngine.reset();
|
||||
if (InitOrtEngine(ANSCENTER::EngineType::CPU)) {
|
||||
_logger.LogInfo("ANSONNXYOLO::DetectObjects",
|
||||
"CPU fallback session created successfully",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "CPU fallback OK");
|
||||
} else {
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
"CPU fallback session creation failed",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
} catch (const std::exception& re) {
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
std::string("CPU fallback exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
// Any DML-fatal HRESULT or DML branded error → rebuild on CPU.
|
||||
// See IsDmlFatalError() in ANSEngineCommon.h for the full set
|
||||
// (887A0005 device-removed, 80004005 generic E_FAIL from
|
||||
// DmlGraphFusionHelper / DmlCommandRecorder, etc.).
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure for full
|
||||
// contract. Tries to rebuild the session on the same
|
||||
// DML EP first; only falls back to CPU on
|
||||
// device-removed (887A0005) or after the retry budget
|
||||
// is exhausted.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::DetectObjects", msg);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -2157,19 +2338,9 @@ namespace ANSCENTER {
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
try {
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
"CPU fallback session creation failed", __FILE__, __LINE__);
|
||||
} catch (...) {}
|
||||
}
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::RunInferencesBatch", msg);
|
||||
return {};
|
||||
}
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
@@ -2184,11 +2355,15 @@ namespace ANSCENTER {
|
||||
try {
|
||||
if (_modelLoading.load()) return {};
|
||||
|
||||
// Snapshot config under brief lock
|
||||
// Snapshot config + engine handle under brief lock.
|
||||
// B2: shared_ptr copy `engine` keeps the underlying ONNXYOLO
|
||||
// alive across the lock-free detectBatch() call even if a
|
||||
// concurrent DML-fatal handler resets m_ortEngine.
|
||||
float probThresh, nmsThresh;
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
{
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjectsBatch");
|
||||
if (!lk.owns_lock()) return {};
|
||||
@@ -2204,13 +2379,20 @@ namespace ANSCENTER {
|
||||
classes = _classes;
|
||||
trackerEnabled = _trackerEnabled;
|
||||
stabilizationEnabled = _stabilizationEnabled;
|
||||
engine = m_ortEngine; // ref-count++
|
||||
}
|
||||
|
||||
// Heavy work outside lock
|
||||
auto batchResults = m_ortEngine->detectBatch(
|
||||
inputImages, classes, probThresh, nmsThresh, numKps);
|
||||
// Heavy work outside lock — dereference local copy, not member.
|
||||
// Option A: same DmlInferenceGuard contract as DetectObjects.
|
||||
std::vector<std::vector<Object>> batchResults;
|
||||
{
|
||||
DmlInferenceGuard dmlGuard(engine->getEngineType(),
|
||||
"ANSONNXYOLO::DetectObjectsBatch");
|
||||
batchResults = engine->detectBatch(
|
||||
inputImages, classes, probThresh, nmsThresh, numKps);
|
||||
}
|
||||
|
||||
const bool isClassification = m_ortEngine->lastBatchWasClassification;
|
||||
const bool isClassification = engine->lastBatchWasClassification;
|
||||
|
||||
for (auto& results : batchResults) {
|
||||
for (auto& obj : results)
|
||||
@@ -2222,24 +2404,16 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Clear DML failure budget on success — see DetectObjects
|
||||
// for the full rationale.
|
||||
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
|
||||
return batchResults;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
try {
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
"CPU fallback session creation failed", __FILE__, __LINE__);
|
||||
} catch (...) {}
|
||||
}
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::DetectObjectsBatch", msg);
|
||||
return {};
|
||||
}
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
|
||||
Reference in New Issue
Block a user