Fix AMD ALPR_OCR and custom model

This commit is contained in:
2026-05-01 07:53:18 +10:00
parent e1f37ec767
commit 91efbb9a8f
25 changed files with 2841 additions and 429 deletions
+196 -23
View File
@@ -2,6 +2,7 @@
#include "include/clipper.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
#include "ANSLicense.h" // ANS_DBG
#include <opencv2/imgproc.hpp>
#include <iostream>
@@ -12,6 +13,130 @@
namespace ANSCENTER {
namespace onnxocr {
// AMD/DirectML latency-based recreate (Path B) — record helper. See
// ONNXOCRRecognizer for the rationale. Arms _slowRecreatePending if
// a Run() exceeded the slow threshold while past warmup and outside
// the cooldown window.
void ONNXOCRDetector::RecordRunLatency(const char* context, double ms) {
if (ms <= kSlowRecreateThresholdMs) return;
if (!_warmedUp) return;
const auto now = std::chrono::steady_clock::now();
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastRecreateAt).count();
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
sinceLast < kRecreateCooldownMs) {
ANS_DBG("ANSONNXDet_run",
"%s: slow Run (%.0f ms) but in cooldown (%lldms)",
context, ms, static_cast<long long>(sinceLast));
return;
}
_slowRecreatePending.store(true, std::memory_order_release);
std::cerr << "[ONNXOCRDetector] " << context
<< ": slow Run (" << ms
<< " ms > threshold) — flagged for proactive recreate\n";
ANS_DBG("ANSONNXDet_run",
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
context, ms, kSlowRecreateThresholdMs);
}
// AMD/DirectML latency-based recreate (Path B) — apply helper. Called
// at entry of public methods. If the slow flag was armed by a prior
// call's RecordRunLatency, swap the session in here so this call uses
// the fresh one.
void ONNXOCRDetector::ApplyPendingSlowRecreate(const char* context) {
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
return;
}
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
<< " (latency-triggered)\n";
ANS_DBG("ANSONNXDet_run",
"%s: proactive recreate on EP=%d", context,
static_cast<int>(currentEp));
try {
Reinitialize(currentEp);
_warmedUp = false;
_lastRecreateAt = std::chrono::steady_clock::now();
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate OK\n";
ANS_DBG("ANSONNXDet_run", "%s: proactive recreate OK", context);
} catch (const std::exception& e) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate failed: " << e.what() << "\n";
ANS_DBG("ANSONNXDet_run",
"%s: proactive recreate failed (%s)", context, e.what());
} catch (...) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": proactive recreate failed (unknown)\n";
}
}
// AMD/DirectML self-heal helper. See ONNXOCRRecognizer for the full
// rationale. On any Run() failure: recreate session on the same EP
// first (transient driver hiccup recovery — the common case for AMD
// DML 0x80004005 E_FAIL); on a second failure, fall back to CPU EP.
// The current call's result is already lost — caller returns empty;
// the recovered session is used by the NEXT Detect call.
void ONNXOCRDetector::HandleRunFailureAndRecreate(const char* context,
const std::exception& e) {
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRDetector] " << context
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
<< ": " << e.what() << " — recreating session\n";
ANS_DBG("ANSONNXDet_run",
"%s: Run failed EP=%d msg=%.200s — recreating",
context, static_cast<int>(currentEp), e.what());
// Option A: trip process-wide DML serial mode on AMD exception
// (idempotent, no-op on Intel/NVIDIA/CPU).
if (currentEp == EngineType::AMD_GPU) {
ANSCENTER::DmlSerialState::TripFromException(
context, e.what());
}
if (currentEp != EngineType::CPU) {
try {
Reinitialize(currentEp);
_warmedUp = false;
std::cerr << "[ONNXOCRDetector] " << context
<< ": session recreated on EP=" << static_cast<int>(currentEp) << "\n";
ANS_DBG("ANSONNXDet_run",
"%s: session recreated on EP=%d", context,
static_cast<int>(currentEp));
return;
} catch (const std::exception& reErr) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": same-EP recreate failed: " << reErr.what()
<< " — falling back to CPU\n";
ANS_DBG("ANSONNXDet_run",
"%s: same-EP recreate failed (%s); falling back to CPU",
context, reErr.what());
} catch (...) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
}
}
try {
Reinitialize(EngineType::CPU);
_warmedUp = false;
std::cerr << "[ONNXOCRDetector] " << context
<< ": fallback session created on CPU\n";
ANS_DBG("ANSONNXDet_run", "%s: CPU fallback OK", context);
} catch (const std::exception& cpuErr) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": CPU fallback ALSO failed: " << cpuErr.what()
<< " — ort_session may be null, future calls will return empty\n";
ANS_DBG("ANSONNXDet_run",
"%s: CPU fallback failed (%s); ort_session may be null",
context, cpuErr.what());
} catch (...) {
std::cerr << "[ONNXOCRDetector] " << context
<< ": CPU fallback ALSO failed (unknown)\n";
}
}
ONNXOCRDetector::ONNXOCRDetector(const std::string& onnx_path, unsigned int num_threads)
: BasicOrtHandler(onnx_path, num_threads) {
}
@@ -55,6 +180,11 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
return {};
}
// Path B (AMD/DML): apply pending proactive recreate before constructing
// any tensors. See ONNXOCRRecognizer for the full rationale.
ApplyPendingSlowRecreate("Detect");
if (!ort_session) return {}; // recreate may have failed; bail safely
// Try to get full-resolution image from NV12 frame (same pattern as ANSONNXYOLO)
cv::Mat inferenceImage = srcImage;
float bgrScaleX = 1.0f, bgrScaleY = 1.0f;
@@ -89,36 +219,60 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
// Run inference
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
// Run inference + post-process inside try/catch. AMD/DirectML
// self-heal: on Run() failure (typically 0x80004005 E_FAIL from a
// transient DML driver hiccup) we recreate the session for the next
// call (same EP first, CPU as last resort). This call's result is
// lost — return empty boxes; ALPR pipeline tolerates a dropped
// detection frame, the tracker recovers. Latency is also recorded
// for the Path B proactive-recreate trigger.
try {
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRDetector::Detect");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
RecordRunLatency("Detect", runMs);
// Get output data
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
// Get output data
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int outH = static_cast<int>(outputShape[2]);
int outW = static_cast<int>(outputShape[3]);
int outH = static_cast<int>(outputShape[2]);
int outW = static_cast<int>(outputShape[3]);
// Postprocess — detection coords are relative to inferenceImage (full-res),
// then scaled back to srcImage (display-res) coordinates
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
inferenceImage.rows, inferenceImage.cols,
dbThresh, boxThresh, unclipRatio, useDilation);
// Postprocess — detection coords are relative to inferenceImage (full-res),
// then scaled back to srcImage (display-res) coordinates
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
inferenceImage.rows, inferenceImage.cols,
dbThresh, boxThresh, unclipRatio, useDilation);
// Rescale box coordinates from full-res to display-res
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
for (auto& box : boxes) {
for (auto& pt : box.points) {
pt.x *= bgrScaleX;
pt.y *= bgrScaleY;
// Rescale box coordinates from full-res to display-res
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
for (auto& box : boxes) {
for (auto& pt : box.points) {
pt.x *= bgrScaleX;
pt.y *= bgrScaleY;
}
}
}
}
return boxes;
return boxes;
}
catch (const Ort::Exception& e) {
HandleRunFailureAndRecreate("Detect", e);
return {};
}
}
std::vector<float> ONNXOCRDetector::Preprocess(const cv::Mat& srcImage, int maxSideLen,
@@ -323,6 +477,8 @@ void ONNXOCRDetector::Warmup() {
// multiple-of-32 rounding. cuDNN caches the algorithm for this
// shape so the first real inference doesn't pay the picker cost.
constexpr int kWarmupSide = 320;
bool warmupFailed = false;
std::string warmupErr;
try {
cv::Mat dummy(kWarmupSide, kWarmupSide, CV_8UC3, cv::Scalar(128, 128, 128));
cv::Mat dummyF;
@@ -347,6 +503,23 @@ void ONNXOCRDetector::Warmup() {
}
catch (const Ort::Exception& e) {
std::cerr << "[ONNXOCRDetector] Warmup failed: " << e.what() << std::endl;
warmupFailed = true;
warmupErr = e.what();
}
// AMD/DML self-heal: if warmup itself threw, the session is dead and
// every Detect() will fail. Recreate now (same EP first, CPU as
// fallback) so production traffic can still serve. Leave _warmedUp
// = false so the recovered session warms up on its first real call.
if (warmupFailed) {
ANS_DBG("ANSONNXDet_run",
"Warmup failed (%.200s); triggering self-heal",
warmupErr.c_str());
try {
std::runtime_error proxy(warmupErr);
HandleRunFailureAndRecreate("Warmup", proxy);
} catch (...) {}
return; // do NOT set _warmedUp; recovered session warms on first real Detect
}
_warmedUp = true;
}
@@ -4,6 +4,8 @@
#include "ONNXEngine.h"
#include <vector>
#include <mutex>
#include <atomic>
#include <chrono>
namespace ANSCENTER {
namespace onnxocr {
@@ -53,6 +55,25 @@ private:
// Expand 4-point box using Clipper offset
std::vector<cv::Point2f> UnclipPolygon(const std::array<cv::Point2f, 4>& box, float unclipRatio);
// AMD/DirectML self-heal helper — see ONNXOCRRecognizer for full
// contract. On Run() failure: recreate session on the same EP first
// (transient hiccup recovery), CPU fallback as last resort. Resets
// _warmedUp so the new session gets re-warmed on next Detect call.
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
// AMD/DirectML latency-based self-heal (Path B) — see ONNXOCRRecognizer
// for the full rationale. Detector is observed to hit the same
// slow-degradation pattern as the recognizer after a sibling DML
// session takes a fatal: Run() doesn't throw, just gets slow.
// 2 s threshold catches degraded individual Run() (healthy is 50–500 ms);
// 60 s cooldown prevents thrash on permanently broken driver state.
static constexpr double kSlowRecreateThresholdMs = 2000.0;
static constexpr int64_t kRecreateCooldownMs = 60000;
std::atomic<bool> _slowRecreatePending{false};
std::chrono::steady_clock::time_point _lastRecreateAt{};
void RecordRunLatency(const char* context, double ms);
void ApplyPendingSlowRecreate(const char* context);
std::mutex _mutex;
};
+274 -15
View File
@@ -113,6 +113,165 @@ static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
return padded;
}
// AMD/DirectML latency-based recreate (Path B) — record helper.
//
// Called after every successful production-path Run(). If the call
// took longer than kSlowRecreateThresholdMs (5 s default), and the
// session has finished its warmup, and we're outside the cooldown
// window, arm _slowRecreatePending so the NEXT public method will
// perform a session swap before its own Run().
//
// Doesn't actually do the swap here — that has to happen at a clean
// boundary (no in-flight tensors, no held output Ort::Values), which
// is precisely the start of the next public method. Calling
// Reinitialize from inside the middle of an inference path while
// holding output_node_names / inputTensor / outputTensors references
// would invalidate them under our feet.
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
if (ms <= kSlowRecreateThresholdMs) return;
if (!_warmedUp) return; // first-shape compile is legitimately slow
// Cooldown gate — if we recreated recently, hold off. Avoids a
// recreate loop if the underlying issue is permanent (e.g. driver
// truly broken) rather than a transient hiccup.
const auto now = std::chrono::steady_clock::now();
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastRecreateAt).count();
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
sinceLast < kRecreateCooldownMs) {
ANS_DBG("ANSONNXRec_run",
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
context, ms, static_cast<long long>(sinceLast));
return;
}
_slowRecreatePending.store(true, std::memory_order_release);
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": slow Run (" << ms
<< " ms > threshold) — flagged for proactive session recreate\n";
ANS_DBG("ANSONNXRec_run",
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
context, ms, kSlowRecreateThresholdMs);
}
// AMD/DirectML latency-based recreate (Path B) — apply helper.
//
// Called at the entry of every public method (Recognize,
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
// armed by a previous call's RecordRunLatency, do the session swap
// here — before any of this call's tensors are constructed. Caller
// already holds _mutex, so the new ort_session is exclusively ours
// for the rest of this call.
//
// On Reinitialize failure we leave _slowRecreatePending=false so we
// don't loop forever; the existing exception-based HandleRunFailure-
// AndRecreate path will fall back to CPU on the next actual throw.
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
return;
}
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
<< " (latency-triggered)\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate on EP=%d (latency-triggered)",
context, static_cast<int>(currentEp));
try {
Reinitialize(currentEp);
_warmedUp = false;
_lastRecreateAt = std::chrono::steady_clock::now();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate succeeded; next call retries on fresh session\n";
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
} catch (const std::exception& e) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed: " << e.what()
<< " — keeping existing session, exception path will catch real failures\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate failed (%s)", context, e.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed (unknown)\n";
}
}
// AMD/DirectML self-heal helper. See header comment for detailed contract.
// In short: on Run() failure, recreate the session on the same EP first
// (transient driver hiccup recovery — the common case); on a second
// failure, fall back to CPU EP. Either path leaves ort_session in a
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
// result is already lost — caller returns empty.
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
const std::exception& e) {
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
<< ": " << e.what() << " — recreating session\n";
ANS_DBG("ANSONNXRec_run",
"%s: Run failed EP=%d msg=%.200s — recreating",
context, static_cast<int>(currentEp), e.what());
// Option A: a Run() exception on a DML session is the strongest
// signal the AMD driver can't sustain parallel inference. Trip
// process-wide DML serial mode so future inference calls (from
// any module) take the global mutex. Idempotent — only the
// first caller per process flips the flag. No-op on
// Intel/NVIDIA/CPU.
if (currentEp == EngineType::AMD_GPU) {
ANSCENTER::DmlSerialState::TripFromException(
context, e.what());
}
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
// are transient driver hiccups; a fresh session typically works. This
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
// throw from Run() under normal load, so this path is rarely taken.
if (currentEp != EngineType::CPU) {
try {
Reinitialize(currentEp);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": session recreated on EP=" << static_cast<int>(currentEp)
<< " — next call will retry on GPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: session recreated on EP=%d (next call retries on GPU)",
context, static_cast<int>(currentEp));
return;
} catch (const std::exception& reCreateErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed: " << reCreateErr.what()
<< " — falling back to CPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: same-EP recreate failed (%s); falling back to CPU",
context, reCreateErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
}
}
// Step 2: last resort — recreate on CPU. Slower but always works
// when the model itself is valid. Caller's next call uses CPU EP.
try {
Reinitialize(EngineType::CPU);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": fallback session created on CPU\n";
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
} catch (const std::exception& cpuErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed: " << cpuErr.what()
<< " — ort_session likely null, future calls will return empty\n";
ANS_DBG("ANSONNXRec_run",
"%s: CPU fallback failed (%s); ort_session may be null",
context, cpuErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed (unknown exception)\n";
}
}
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
std::lock_guard<std::mutex> lock(_mutex);
@@ -120,6 +279,12 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
return {};
}
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
// are built — so the rest of this method runs on a fresh session.
ApplyPendingSlowRecreate("Recognize");
if (!ort_session) return {}; // recreate could have failed; bail safely
try {
// Step 1: aspect-preserving resize to height=imgH_, width capped
// at imgMaxW_. Then round resized width up to the next bucket.
@@ -133,10 +298,24 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::Recognize");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
// Latency-based recreate (Path B): arms the flag for the NEXT call
// if this one was abnormally slow.
RecordRunLatency("Recognize", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -147,7 +326,11 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
return CTCDecode(outputData, seqLen, numClasses);
}
catch (const Ort::Exception& e) {
std::cerr << "[ONNXOCRRecognizer] Inference failed: " << e.what() << std::endl;
// AMD/DML self-heal: recreate session for next call (same EP first,
// CPU fallback as last resort). This call's output is lost — return
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
// tracker migrates plate text across frames.
HandleRunFailureAndRecreate("Recognize", e);
return {};
}
}
@@ -158,6 +341,13 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
std::vector<TextLine>& out) {
if (crops.empty()) return;
// Path B (AMD/DML): apply pending proactive recreate before any tensors
// are constructed for this batch. RecognizeBatch holds _mutex around
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
// same lock the caller already owns.
ApplyPendingSlowRecreate("RunBatchAtWidth");
if (!ort_session) return; // recreate may have failed; abort safely
try {
// Real number of crops vs. padded batch size submitted to ORT.
// Padding to the ladder keeps the recognizer's input-shape set
@@ -189,10 +379,22 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
*memory_info_handler, batchInput.data(), batchInput.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
RecordRunLatency("RunBatchAtWidth", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -215,11 +417,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
}
}
catch (const Ort::Exception& e) {
// ORT will throw if the model doesn't support a batch dimension > 1.
// Fall back to per-image inference for this group.
// The original recovery path here covered "model doesn't support
// batch dim >1" — a one-shot decision: drop to single-image and
// try each crop. We keep that, but ALSO add AMD/DML self-heal
// on top: if the per-image fallback ALSO throws, we recreate the
// session (same EP first, then CPU) so the next caller gets a
// healthy session. The current call's results stay empty for
// any crops we couldn't decode — caller tolerates this.
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
<< bucketW << " (" << e.what()
<< ") — falling back to single-image path." << std::endl;
bool anyPerImageFailed = false;
std::string lastPerImageErr;
for (size_t i = 0; i < crops.size(); ++i) {
// Direct call (we already hold _mutex via the public RecognizeBatch
// wrapper). Replicate the single-image preprocessing here to avoid
@@ -232,10 +441,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
std::vector<Ort::Value> outputTensors;
{
// Option A: per-image fallback path also goes
// through the DML serialization gate.
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType,
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
float* outData = outputTensors[0].GetTensorMutableData<float>();
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int seqLen = static_cast<int>(outShape[1]);
@@ -245,6 +462,20 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
<< e2.what() << std::endl;
out[origIndices[i]] = {};
anyPerImageFailed = true;
lastPerImageErr = e2.what();
}
}
// If even the per-image fallback path is throwing, the session is
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
// session (same EP first, CPU as last resort).
if (anyPerImageFailed) {
try {
std::runtime_error proxy(lastPerImageErr);
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
} catch (...) {
// HandleRunFailureAndRecreate already logs; swallow.
}
}
}
@@ -323,10 +554,13 @@ void ONNXOCRRecognizer::Warmup() {
// plates pay a one-time spike on first occurrence of batch=16/24.
auto totalT0 = std::chrono::high_resolution_clock::now();
int warmedShapes = 0;
int totalShapes = 0;
std::string lastWarmupErr;
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
const int batchN = kRecWarmupBatchLadder[bs];
for (int b = 0; b < kRecNumBuckets; ++b) {
const int bucketW = kRecBucketWidths[b];
++totalShapes;
try {
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
@@ -361,6 +595,7 @@ void ONNXOCRRecognizer::Warmup() {
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
<< batchN << " bucketW=" << bucketW << ": "
<< e.what() << std::endl;
lastWarmupErr = e.what();
}
}
}
@@ -368,6 +603,30 @@ void ONNXOCRRecognizer::Warmup() {
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
<< " shapes in " << totalMs << " ms total" << std::endl;
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
// production traffic will hit Run() failures on every call. Recreate
// it now (same EP first, CPU as fallback) so the pipeline can still
// serve. We deliberately keep _warmedUp = false in this path; the
// next RecognizeBatch will re-trigger Warmup on the recovered session.
//
// We do NOT trigger self-heal on partial failure — losing 2 of 16
// shapes is normal (e.g. some bucket dims aren't supported by all
// EPs); the per-shape catch already guards against that.
if (warmedShapes == 0 && totalShapes > 0) {
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
<< totalShapes << " shapes failed — triggering self-heal\n";
ANS_DBG("ANSONNXRec_run",
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
totalShapes, lastWarmupErr.c_str());
try {
std::runtime_error proxy(lastWarmupErr);
HandleRunFailureAndRecreate("Warmup", proxy);
} catch (...) {}
// Don't set _warmedUp = true; let the recovered session warm up
// itself on the next caller's invocation.
return;
}
_warmedUp = true;
}
@@ -5,6 +5,8 @@
#include <vector>
#include <string>
#include <mutex>
#include <atomic>
#include <chrono>
namespace ANSCENTER {
namespace onnxocr {
@@ -51,6 +53,62 @@ private:
// CTC greedy decode
TextLine CTCDecode(const float* outputData, int seqLen, int numClasses);
// AMD/DirectML self-heal helper. Called by Recognize / RunBatchAtWidth /
// Warmup when ort_session->Run() throws an Ort::Exception. Tries to
// recreate the session on the same EP first (transient driver hiccup
// recovery — the common case for DML 0x80004005 E_FAIL); if that fails,
// falls back to CPU EP. On full failure leaves ort_session in whatever
// state Reinitialize left it (possibly nullptr); the next Recognize
// call will short-circuit on the !ort_session check and return empty.
// Resets _warmedUp so the new session gets re-warmed on first use.
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
// ----------------------------------------------------------------
// AMD/DirectML latency-based self-heal (Path B).
//
// Observed AMD failure mode: after a transient driver event (e.g.
// helmet detector hits DML E_FAIL and CPU-falls-back), DML stays in
// a degraded state. OCR Run() calls do NOT throw, but their latency
// jumps from 50–200 ms to 1000–7000 ms, often rising over time. The
// exception-based self-heal above never fires because there's no
// exception. This proactive path observes the latency and forces a
// session recreate when a Run() exceeds a sanity threshold — the
// fresh session typically returns to baseline latency.
//
// How it works:
// 1. Each public method records its Run() duration.
// 2. If duration > kSlowRecreateThresholdMs AND we're past warmup
// AND we're past the cooldown window, set _slowRecreatePending.
// 3. The NEXT call to a public method checks the flag at entry; if
// set, it calls Reinitialize(currentEp) BEFORE its Run() — so
// the call that sees the flag uses a fresh session and returns
// to fast latency.
// 4. After a recreate, _lastRecreateMs starts the cooldown timer.
//
// Threshold tuning: 2 s catches the AMD-degraded individual Run()
// pattern observed in production (1–2 s per OCR Run() when DML is
// sick). Healthy individual Run() is 50–500 ms, so 2 s leaves
// comfortable headroom. First-shape compiles on OpenVINO can hit
// ~1.7 s but those happen during Warmup, which is gated out below
// via the _warmedUp check. 60 s cooldown prevents thrash if the
// driver is in a permanently bad state — at most one recreate per
// minute per session.
// ----------------------------------------------------------------
static constexpr double kSlowRecreateThresholdMs = 2000.0;
static constexpr int64_t kRecreateCooldownMs = 60000;
// Note: a slow Run() detected by RecordRunLatency arms this flag;
// ApplyPendingSlowRecreate (called at every public-method entry)
// performs the actual Reinitialize and clears the flag. std::atomic
// is used so the read at entry is lock-free; the write happens under
// the same _mutex the public methods already hold, so the only race
// we're guarding is "another public method picked up the flag first."
std::atomic<bool> _slowRecreatePending{false};
std::chrono::steady_clock::time_point _lastRecreateAt{};
void RecordRunLatency(const char* context, double ms);
void ApplyPendingSlowRecreate(const char* context);
std::vector<std::string> keys_;
int imgH_ = kRecImgH;
int imgMaxW_ = kRecImgMaxW;