Fix AMD ALPR_OCR and custom model

This commit is contained in:
2026-05-01 07:53:18 +10:00
parent e1f37ec767
commit 91efbb9a8f
25 changed files with 2841 additions and 429 deletions
+274 -15
View File
@@ -113,6 +113,165 @@ static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
return padded;
}
// AMD/DirectML latency-based recreate (Path B) — record helper.
//
// Called after every successful production-path Run(). If the call
// took longer than kSlowRecreateThresholdMs (5 s default), and the
// session has finished its warmup, and we're outside the cooldown
// window, arm _slowRecreatePending so the NEXT public method will
// perform a session swap before its own Run().
//
// Doesn't actually do the swap here — that has to happen at a clean
// boundary (no in-flight tensors, no held output Ort::Values), which
// is precisely the start of the next public method. Calling
// Reinitialize from inside the middle of an inference path while
// holding output_node_names / inputTensor / outputTensors references
// would invalidate them under our feet.
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
if (ms <= kSlowRecreateThresholdMs) return;
if (!_warmedUp) return; // first-shape compile is legitimately slow
// Cooldown gate — if we recreated recently, hold off. Avoids a
// recreate loop if the underlying issue is permanent (e.g. driver
// truly broken) rather than a transient hiccup.
const auto now = std::chrono::steady_clock::now();
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastRecreateAt).count();
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
sinceLast < kRecreateCooldownMs) {
ANS_DBG("ANSONNXRec_run",
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
context, ms, static_cast<long long>(sinceLast));
return;
}
_slowRecreatePending.store(true, std::memory_order_release);
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": slow Run (" << ms
<< " ms > threshold) — flagged for proactive session recreate\n";
ANS_DBG("ANSONNXRec_run",
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
context, ms, kSlowRecreateThresholdMs);
}
// AMD/DirectML latency-based recreate (Path B) — apply helper.
//
// Called at the entry of every public method (Recognize,
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
// armed by a previous call's RecordRunLatency, do the session swap
// here — before any of this call's tensors are constructed. Caller
// already holds _mutex, so the new ort_session is exclusively ours
// for the rest of this call.
//
// On Reinitialize failure we leave _slowRecreatePending=false so we
// don't loop forever; the existing exception-based HandleRunFailure-
// AndRecreate path will fall back to CPU on the next actual throw.
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
return;
}
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
<< " (latency-triggered)\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate on EP=%d (latency-triggered)",
context, static_cast<int>(currentEp));
try {
Reinitialize(currentEp);
_warmedUp = false;
_lastRecreateAt = std::chrono::steady_clock::now();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate succeeded; next call retries on fresh session\n";
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
} catch (const std::exception& e) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed: " << e.what()
<< " — keeping existing session, exception path will catch real failures\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate failed (%s)", context, e.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed (unknown)\n";
}
}
// AMD/DirectML self-heal helper. See header comment for detailed contract.
// In short: on Run() failure, recreate the session on the same EP first
// (transient driver hiccup recovery — the common case); on a second
// failure, fall back to CPU EP. Either path leaves ort_session in a
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
// result is already lost — caller returns empty.
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
const std::exception& e) {
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
<< ": " << e.what() << " — recreating session\n";
ANS_DBG("ANSONNXRec_run",
"%s: Run failed EP=%d msg=%.200s — recreating",
context, static_cast<int>(currentEp), e.what());
// Option A: a Run() exception on a DML session is the strongest
// signal the AMD driver can't sustain parallel inference. Trip
// process-wide DML serial mode so future inference calls (from
// any module) take the global mutex. Idempotent — only the
// first caller per process flips the flag. No-op on
// Intel/NVIDIA/CPU.
if (currentEp == EngineType::AMD_GPU) {
ANSCENTER::DmlSerialState::TripFromException(
context, e.what());
}
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
// are transient driver hiccups; a fresh session typically works. This
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
// throw from Run() under normal load, so this path is rarely taken.
if (currentEp != EngineType::CPU) {
try {
Reinitialize(currentEp);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": session recreated on EP=" << static_cast<int>(currentEp)
<< " — next call will retry on GPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: session recreated on EP=%d (next call retries on GPU)",
context, static_cast<int>(currentEp));
return;
} catch (const std::exception& reCreateErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed: " << reCreateErr.what()
<< " — falling back to CPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: same-EP recreate failed (%s); falling back to CPU",
context, reCreateErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
}
}
// Step 2: last resort — recreate on CPU. Slower but always works
// when the model itself is valid. Caller's next call uses CPU EP.
try {
Reinitialize(EngineType::CPU);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": fallback session created on CPU\n";
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
} catch (const std::exception& cpuErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed: " << cpuErr.what()
<< " — ort_session likely null, future calls will return empty\n";
ANS_DBG("ANSONNXRec_run",
"%s: CPU fallback failed (%s); ort_session may be null",
context, cpuErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed (unknown exception)\n";
}
}
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
std::lock_guard<std::mutex> lock(_mutex);
@@ -120,6 +279,12 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
return {};
}
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
// are built — so the rest of this method runs on a fresh session.
ApplyPendingSlowRecreate("Recognize");
if (!ort_session) return {}; // recreate could have failed; bail safely
try {
// Step 1: aspect-preserving resize to height=imgH_, width capped
// at imgMaxW_. Then round resized width up to the next bucket.
@@ -133,10 +298,24 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::Recognize");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
// Latency-based recreate (Path B): arms the flag for the NEXT call
// if this one was abnormally slow.
RecordRunLatency("Recognize", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -147,7 +326,11 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
return CTCDecode(outputData, seqLen, numClasses);
}
catch (const Ort::Exception& e) {
std::cerr << "[ONNXOCRRecognizer] Inference failed: " << e.what() << std::endl;
// AMD/DML self-heal: recreate session for next call (same EP first,
// CPU fallback as last resort). This call's output is lost — return
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
// tracker migrates plate text across frames.
HandleRunFailureAndRecreate("Recognize", e);
return {};
}
}
@@ -158,6 +341,13 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
std::vector<TextLine>& out) {
if (crops.empty()) return;
// Path B (AMD/DML): apply pending proactive recreate before any tensors
// are constructed for this batch. RecognizeBatch holds _mutex around
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
// same lock the caller already owns.
ApplyPendingSlowRecreate("RunBatchAtWidth");
if (!ort_session) return; // recreate may have failed; abort safely
try {
// Real number of crops vs. padded batch size submitted to ORT.
// Padding to the ladder keeps the recognizer's input-shape set
@@ -189,10 +379,22 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
*memory_info_handler, batchInput.data(), batchInput.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
RecordRunLatency("RunBatchAtWidth", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -215,11 +417,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
}
}
catch (const Ort::Exception& e) {
// ORT will throw if the model doesn't support a batch dimension > 1.
// Fall back to per-image inference for this group.
// The original recovery path here covered "model doesn't support
// batch dim >1" — a one-shot decision: drop to single-image and
// try each crop. We keep that, but ALSO add AMD/DML self-heal
// on top: if the per-image fallback ALSO throws, we recreate the
// session (same EP first, then CPU) so the next caller gets a
// healthy session. The current call's results stay empty for
// any crops we couldn't decode — caller tolerates this.
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
<< bucketW << " (" << e.what()
<< ") — falling back to single-image path." << std::endl;
bool anyPerImageFailed = false;
std::string lastPerImageErr;
for (size_t i = 0; i < crops.size(); ++i) {
// Direct call (we already hold _mutex via the public RecognizeBatch
// wrapper). Replicate the single-image preprocessing here to avoid
@@ -232,10 +441,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
std::vector<Ort::Value> outputTensors;
{
// Option A: per-image fallback path also goes
// through the DML serialization gate.
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType,
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
float* outData = outputTensors[0].GetTensorMutableData<float>();
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int seqLen = static_cast<int>(outShape[1]);
@@ -245,6 +462,20 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
<< e2.what() << std::endl;
out[origIndices[i]] = {};
anyPerImageFailed = true;
lastPerImageErr = e2.what();
}
}
// If even the per-image fallback path is throwing, the session is
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
// session (same EP first, CPU as last resort).
if (anyPerImageFailed) {
try {
std::runtime_error proxy(lastPerImageErr);
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
} catch (...) {
// HandleRunFailureAndRecreate already logs; swallow.
}
}
}
@@ -323,10 +554,13 @@ void ONNXOCRRecognizer::Warmup() {
// plates pay a one-time spike on first occurrence of batch=16/24.
auto totalT0 = std::chrono::high_resolution_clock::now();
int warmedShapes = 0;
int totalShapes = 0;
std::string lastWarmupErr;
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
const int batchN = kRecWarmupBatchLadder[bs];
for (int b = 0; b < kRecNumBuckets; ++b) {
const int bucketW = kRecBucketWidths[b];
++totalShapes;
try {
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
@@ -361,6 +595,7 @@ void ONNXOCRRecognizer::Warmup() {
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
<< batchN << " bucketW=" << bucketW << ": "
<< e.what() << std::endl;
lastWarmupErr = e.what();
}
}
}
@@ -368,6 +603,30 @@ void ONNXOCRRecognizer::Warmup() {
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
<< " shapes in " << totalMs << " ms total" << std::endl;
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
// production traffic will hit Run() failures on every call. Recreate
// it now (same EP first, CPU as fallback) so the pipeline can still
// serve. We deliberately keep _warmedUp = false in this path; the
// next RecognizeBatch will re-trigger Warmup on the recovered session.
//
// We do NOT trigger self-heal on partial failure — losing 2 of 16
// shapes is normal (e.g. some bucket dims aren't supported by all
// EPs); the per-shape catch already guards against that.
if (warmedShapes == 0 && totalShapes > 0) {
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
<< totalShapes << " shapes failed — triggering self-heal\n";
ANS_DBG("ANSONNXRec_run",
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
totalShapes, lastWarmupErr.c_str());
try {
std::runtime_error proxy(lastWarmupErr);
HandleRunFailureAndRecreate("Warmup", proxy);
} catch (...) {}
// Don't set _warmedUp = true; let the recovered session warm up
// itself on the next caller's invocation.
return;
}
_warmedUp = true;
}