Fix AMD ALPR_OCR and custom model
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
#include "include/clipper.h"
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "NV12PreprocessHelper.h"
|
||||
#include "ANSLicense.h" // ANS_DBG
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <iostream>
|
||||
@@ -12,6 +13,130 @@
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — record helper. See
|
||||
// ONNXOCRRecognizer for the rationale. Arms _slowRecreatePending if
|
||||
// a Run() exceeded the slow threshold while past warmup and outside
|
||||
// the cooldown window.
|
||||
void ONNXOCRDetector::RecordRunLatency(const char* context, double ms) {
|
||||
if (ms <= kSlowRecreateThresholdMs) return;
|
||||
if (!_warmedUp) return;
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastRecreateAt).count();
|
||||
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
|
||||
sinceLast < kRecreateCooldownMs) {
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: slow Run (%.0f ms) but in cooldown (%lldms)",
|
||||
context, ms, static_cast<long long>(sinceLast));
|
||||
return;
|
||||
}
|
||||
_slowRecreatePending.store(true, std::memory_order_release);
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": slow Run (" << ms
|
||||
<< " ms > threshold) — flagged for proactive recreate\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
|
||||
context, ms, kSlowRecreateThresholdMs);
|
||||
}
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — apply helper. Called
|
||||
// at entry of public methods. If the slow flag was armed by a prior
|
||||
// call's RecordRunLatency, swap the session in here so this call uses
|
||||
// the fresh one.
|
||||
void ONNXOCRDetector::ApplyPendingSlowRecreate(const char* context) {
|
||||
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
|
||||
return;
|
||||
}
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
|
||||
<< " (latency-triggered)\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: proactive recreate on EP=%d", context,
|
||||
static_cast<int>(currentEp));
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
_lastRecreateAt = std::chrono::steady_clock::now();
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate OK\n";
|
||||
ANS_DBG("ANSONNXDet_run", "%s: proactive recreate OK", context);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate failed: " << e.what() << "\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: proactive recreate failed (%s)", context, e.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": proactive recreate failed (unknown)\n";
|
||||
}
|
||||
}
|
||||
|
||||
// AMD/DirectML self-heal helper. See ONNXOCRRecognizer for the full
|
||||
// rationale. On any Run() failure: recreate session on the same EP
|
||||
// first (transient driver hiccup recovery — the common case for AMD
|
||||
// DML 0x80004005 E_FAIL); on a second failure, fall back to CPU EP.
|
||||
// The current call's result is already lost — caller returns empty;
|
||||
// the recovered session is used by the NEXT Detect call.
|
||||
void ONNXOCRDetector::HandleRunFailureAndRecreate(const char* context,
|
||||
const std::exception& e) {
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
|
||||
<< ": " << e.what() << " — recreating session\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: Run failed EP=%d msg=%.200s — recreating",
|
||||
context, static_cast<int>(currentEp), e.what());
|
||||
|
||||
// Option A: trip process-wide DML serial mode on AMD exception
|
||||
// (idempotent, no-op on Intel/NVIDIA/CPU).
|
||||
if (currentEp == EngineType::AMD_GPU) {
|
||||
ANSCENTER::DmlSerialState::TripFromException(
|
||||
context, e.what());
|
||||
}
|
||||
|
||||
if (currentEp != EngineType::CPU) {
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": session recreated on EP=" << static_cast<int>(currentEp) << "\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: session recreated on EP=%d", context,
|
||||
static_cast<int>(currentEp));
|
||||
return;
|
||||
} catch (const std::exception& reErr) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": same-EP recreate failed: " << reErr.what()
|
||||
<< " — falling back to CPU\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: same-EP recreate failed (%s); falling back to CPU",
|
||||
context, reErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
Reinitialize(EngineType::CPU);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": fallback session created on CPU\n";
|
||||
ANS_DBG("ANSONNXDet_run", "%s: CPU fallback OK", context);
|
||||
} catch (const std::exception& cpuErr) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": CPU fallback ALSO failed: " << cpuErr.what()
|
||||
<< " — ort_session may be null, future calls will return empty\n";
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"%s: CPU fallback failed (%s); ort_session may be null",
|
||||
context, cpuErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRDetector] " << context
|
||||
<< ": CPU fallback ALSO failed (unknown)\n";
|
||||
}
|
||||
}
|
||||
|
||||
ONNXOCRDetector::ONNXOCRDetector(const std::string& onnx_path, unsigned int num_threads)
|
||||
: BasicOrtHandler(onnx_path, num_threads) {
|
||||
}
|
||||
@@ -55,6 +180,11 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
|
||||
return {};
|
||||
}
|
||||
|
||||
// Path B (AMD/DML): apply pending proactive recreate before constructing
|
||||
// any tensors. See ONNXOCRRecognizer for the full rationale.
|
||||
ApplyPendingSlowRecreate("Detect");
|
||||
if (!ort_session) return {}; // recreate may have failed; bail safely
|
||||
|
||||
// Try to get full-resolution image from NV12 frame (same pattern as ANSONNXYOLO)
|
||||
cv::Mat inferenceImage = srcImage;
|
||||
float bgrScaleX = 1.0f, bgrScaleY = 1.0f;
|
||||
@@ -89,36 +219,60 @@ std::vector<TextBox> ONNXOCRDetector::Detect(const cv::Mat& srcImage,
|
||||
*memory_info_handler, inputData.data(), inputData.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
|
||||
// Run inference
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
// Run inference + post-process inside try/catch. AMD/DirectML
|
||||
// self-heal: on Run() failure (typically 0x80004005 E_FAIL from a
|
||||
// transient DML driver hiccup) we recreate the session for the next
|
||||
// call (same EP first, CPU as last resort). This call's result is
|
||||
// lost — return empty boxes; ALPR pipeline tolerates a dropped
|
||||
// detection frame, the tracker recovers. Latency is also recorded
|
||||
// for the Path B proactive-recreate trigger.
|
||||
try {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: serialize DML inference once per-process
|
||||
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType, "ONNXOCRDetector::Detect");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double runMs =
|
||||
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
||||
RecordRunLatency("Detect", runMs);
|
||||
|
||||
// Get output data
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
// Get output data
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
|
||||
int outH = static_cast<int>(outputShape[2]);
|
||||
int outW = static_cast<int>(outputShape[3]);
|
||||
int outH = static_cast<int>(outputShape[2]);
|
||||
int outW = static_cast<int>(outputShape[3]);
|
||||
|
||||
// Postprocess — detection coords are relative to inferenceImage (full-res),
|
||||
// then scaled back to srcImage (display-res) coordinates
|
||||
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
|
||||
inferenceImage.rows, inferenceImage.cols,
|
||||
dbThresh, boxThresh, unclipRatio, useDilation);
|
||||
// Postprocess — detection coords are relative to inferenceImage (full-res),
|
||||
// then scaled back to srcImage (display-res) coordinates
|
||||
auto boxes = Postprocess(outputData, outH, outW, ratioH, ratioW,
|
||||
inferenceImage.rows, inferenceImage.cols,
|
||||
dbThresh, boxThresh, unclipRatio, useDilation);
|
||||
|
||||
// Rescale box coordinates from full-res to display-res
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
for (auto& box : boxes) {
|
||||
for (auto& pt : box.points) {
|
||||
pt.x *= bgrScaleX;
|
||||
pt.y *= bgrScaleY;
|
||||
// Rescale box coordinates from full-res to display-res
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
for (auto& box : boxes) {
|
||||
for (auto& pt : box.points) {
|
||||
pt.x *= bgrScaleX;
|
||||
pt.y *= bgrScaleY;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return boxes;
|
||||
return boxes;
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
HandleRunFailureAndRecreate("Detect", e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> ONNXOCRDetector::Preprocess(const cv::Mat& srcImage, int maxSideLen,
|
||||
@@ -323,6 +477,8 @@ void ONNXOCRDetector::Warmup() {
|
||||
// multiple-of-32 rounding. cuDNN caches the algorithm for this
|
||||
// shape so the first real inference doesn't pay the picker cost.
|
||||
constexpr int kWarmupSide = 320;
|
||||
bool warmupFailed = false;
|
||||
std::string warmupErr;
|
||||
try {
|
||||
cv::Mat dummy(kWarmupSide, kWarmupSide, CV_8UC3, cv::Scalar(128, 128, 128));
|
||||
cv::Mat dummyF;
|
||||
@@ -347,6 +503,23 @@ void ONNXOCRDetector::Warmup() {
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
std::cerr << "[ONNXOCRDetector] Warmup failed: " << e.what() << std::endl;
|
||||
warmupFailed = true;
|
||||
warmupErr = e.what();
|
||||
}
|
||||
|
||||
// AMD/DML self-heal: if warmup itself threw, the session is dead and
|
||||
// every Detect() will fail. Recreate now (same EP first, CPU as
|
||||
// fallback) so production traffic can still serve. Leave _warmedUp
|
||||
// = false so the recovered session warms up on its first real call.
|
||||
if (warmupFailed) {
|
||||
ANS_DBG("ANSONNXDet_run",
|
||||
"Warmup failed (%.200s); triggering self-heal",
|
||||
warmupErr.c_str());
|
||||
try {
|
||||
std::runtime_error proxy(warmupErr);
|
||||
HandleRunFailureAndRecreate("Warmup", proxy);
|
||||
} catch (...) {}
|
||||
return; // do NOT set _warmedUp; recovered session warms on first real Detect
|
||||
}
|
||||
_warmedUp = true;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include "ONNXEngine.h"
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
@@ -53,6 +55,25 @@ private:
|
||||
// Expand 4-point box using Clipper offset
|
||||
std::vector<cv::Point2f> UnclipPolygon(const std::array<cv::Point2f, 4>& box, float unclipRatio);
|
||||
|
||||
// AMD/DirectML self-heal helper — see ONNXOCRRecognizer for full
|
||||
// contract. On Run() failure: recreate session on the same EP first
|
||||
// (transient hiccup recovery), CPU fallback as last resort. Resets
|
||||
// _warmedUp so the new session gets re-warmed on next Detect call.
|
||||
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
|
||||
|
||||
// AMD/DirectML latency-based self-heal (Path B) — see ONNXOCRRecognizer
|
||||
// for the full rationale. Detector is observed to hit the same
|
||||
// slow-degradation pattern as the recognizer after a sibling DML
|
||||
// session takes a fatal: Run() doesn't throw, just gets slow.
|
||||
// 2 s threshold catches degraded individual Run() (healthy is 50–500 ms);
|
||||
// 60 s cooldown prevents thrash on permanently broken driver state.
|
||||
static constexpr double kSlowRecreateThresholdMs = 2000.0;
|
||||
static constexpr int64_t kRecreateCooldownMs = 60000;
|
||||
std::atomic<bool> _slowRecreatePending{false};
|
||||
std::chrono::steady_clock::time_point _lastRecreateAt{};
|
||||
void RecordRunLatency(const char* context, double ms);
|
||||
void ApplyPendingSlowRecreate(const char* context);
|
||||
|
||||
std::mutex _mutex;
|
||||
};
|
||||
|
||||
|
||||
@@ -113,6 +113,165 @@ static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
|
||||
return padded;
|
||||
}
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — record helper.
|
||||
//
|
||||
// Called after every successful production-path Run(). If the call
|
||||
// took longer than kSlowRecreateThresholdMs (5 s default), and the
|
||||
// session has finished its warmup, and we're outside the cooldown
|
||||
// window, arm _slowRecreatePending so the NEXT public method will
|
||||
// perform a session swap before its own Run().
|
||||
//
|
||||
// Doesn't actually do the swap here — that has to happen at a clean
|
||||
// boundary (no in-flight tensors, no held output Ort::Values), which
|
||||
// is precisely the start of the next public method. Calling
|
||||
// Reinitialize from inside the middle of an inference path while
|
||||
// holding output_node_names / inputTensor / outputTensors references
|
||||
// would invalidate them under our feet.
|
||||
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
|
||||
if (ms <= kSlowRecreateThresholdMs) return;
|
||||
if (!_warmedUp) return; // first-shape compile is legitimately slow
|
||||
|
||||
// Cooldown gate — if we recreated recently, hold off. Avoids a
|
||||
// recreate loop if the underlying issue is permanent (e.g. driver
|
||||
// truly broken) rather than a transient hiccup.
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastRecreateAt).count();
|
||||
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
|
||||
sinceLast < kRecreateCooldownMs) {
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
|
||||
context, ms, static_cast<long long>(sinceLast));
|
||||
return;
|
||||
}
|
||||
|
||||
_slowRecreatePending.store(true, std::memory_order_release);
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": slow Run (" << ms
|
||||
<< " ms > threshold) — flagged for proactive session recreate\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
|
||||
context, ms, kSlowRecreateThresholdMs);
|
||||
}
|
||||
|
||||
// AMD/DirectML latency-based recreate (Path B) — apply helper.
|
||||
//
|
||||
// Called at the entry of every public method (Recognize,
|
||||
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
|
||||
// armed by a previous call's RecordRunLatency, do the session swap
|
||||
// here — before any of this call's tensors are constructed. Caller
|
||||
// already holds _mutex, so the new ort_session is exclusively ours
|
||||
// for the rest of this call.
|
||||
//
|
||||
// On Reinitialize failure we leave _slowRecreatePending=false so we
|
||||
// don't loop forever; the existing exception-based HandleRunFailure-
|
||||
// AndRecreate path will fall back to CPU on the next actual throw.
|
||||
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
|
||||
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
|
||||
return;
|
||||
}
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
|
||||
<< " (latency-triggered)\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: proactive recreate on EP=%d (latency-triggered)",
|
||||
context, static_cast<int>(currentEp));
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
_lastRecreateAt = std::chrono::steady_clock::now();
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate succeeded; next call retries on fresh session\n";
|
||||
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate failed: " << e.what()
|
||||
<< " — keeping existing session, exception path will catch real failures\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: proactive recreate failed (%s)", context, e.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": proactive recreate failed (unknown)\n";
|
||||
}
|
||||
}
|
||||
|
||||
// AMD/DirectML self-heal helper. See header comment for detailed contract.
|
||||
// In short: on Run() failure, recreate the session on the same EP first
|
||||
// (transient driver hiccup recovery — the common case); on a second
|
||||
// failure, fall back to CPU EP. Either path leaves ort_session in a
|
||||
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
|
||||
// result is already lost — caller returns empty.
|
||||
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
|
||||
const std::exception& e) {
|
||||
const EngineType currentEp = getEngineType();
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
|
||||
<< ": " << e.what() << " — recreating session\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: Run failed EP=%d msg=%.200s — recreating",
|
||||
context, static_cast<int>(currentEp), e.what());
|
||||
|
||||
// Option A: a Run() exception on a DML session is the strongest
|
||||
// signal the AMD driver can't sustain parallel inference. Trip
|
||||
// process-wide DML serial mode so future inference calls (from
|
||||
// any module) take the global mutex. Idempotent — only the
|
||||
// first caller per process flips the flag. No-op on
|
||||
// Intel/NVIDIA/CPU.
|
||||
if (currentEp == EngineType::AMD_GPU) {
|
||||
ANSCENTER::DmlSerialState::TripFromException(
|
||||
context, e.what());
|
||||
}
|
||||
|
||||
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
|
||||
// are transient driver hiccups; a fresh session typically works. This
|
||||
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
|
||||
// throw from Run() under normal load, so this path is rarely taken.
|
||||
if (currentEp != EngineType::CPU) {
|
||||
try {
|
||||
Reinitialize(currentEp);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": session recreated on EP=" << static_cast<int>(currentEp)
|
||||
<< " — next call will retry on GPU\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: session recreated on EP=%d (next call retries on GPU)",
|
||||
context, static_cast<int>(currentEp));
|
||||
return;
|
||||
} catch (const std::exception& reCreateErr) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": same-EP recreate failed: " << reCreateErr.what()
|
||||
<< " — falling back to CPU\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: same-EP recreate failed (%s); falling back to CPU",
|
||||
context, reCreateErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: last resort — recreate on CPU. Slower but always works
|
||||
// when the model itself is valid. Caller's next call uses CPU EP.
|
||||
try {
|
||||
Reinitialize(EngineType::CPU);
|
||||
_warmedUp = false;
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": fallback session created on CPU\n";
|
||||
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
|
||||
} catch (const std::exception& cpuErr) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": CPU fallback ALSO failed: " << cpuErr.what()
|
||||
<< " — ort_session likely null, future calls will return empty\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"%s: CPU fallback failed (%s); ort_session may be null",
|
||||
context, cpuErr.what());
|
||||
} catch (...) {
|
||||
std::cerr << "[ONNXOCRRecognizer] " << context
|
||||
<< ": CPU fallback ALSO failed (unknown exception)\n";
|
||||
}
|
||||
}
|
||||
|
||||
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
|
||||
@@ -120,6 +279,12 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
|
||||
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
|
||||
// are built — so the rest of this method runs on a fresh session.
|
||||
ApplyPendingSlowRecreate("Recognize");
|
||||
if (!ort_session) return {}; // recreate could have failed; bail safely
|
||||
|
||||
try {
|
||||
// Step 1: aspect-preserving resize to height=imgH_, width capped
|
||||
// at imgMaxW_. Then round resized width up to the next bucket.
|
||||
@@ -133,10 +298,24 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
*memory_info_handler, inputData.data(), inputData.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: serialize DML inference once per-process
|
||||
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType, "ONNXOCRRecognizer::Recognize");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double runMs =
|
||||
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
||||
// Latency-based recreate (Path B): arms the flag for the NEXT call
|
||||
// if this one was abnormally slow.
|
||||
RecordRunLatency("Recognize", runMs);
|
||||
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
@@ -147,7 +326,11 @@ TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
||||
return CTCDecode(outputData, seqLen, numClasses);
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
std::cerr << "[ONNXOCRRecognizer] Inference failed: " << e.what() << std::endl;
|
||||
// AMD/DML self-heal: recreate session for next call (same EP first,
|
||||
// CPU fallback as last resort). This call's output is lost — return
|
||||
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
|
||||
// tracker migrates plate text across frames.
|
||||
HandleRunFailureAndRecreate("Recognize", e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
@@ -158,6 +341,13 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
std::vector<TextLine>& out) {
|
||||
if (crops.empty()) return;
|
||||
|
||||
// Path B (AMD/DML): apply pending proactive recreate before any tensors
|
||||
// are constructed for this batch. RecognizeBatch holds _mutex around
|
||||
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
|
||||
// same lock the caller already owns.
|
||||
ApplyPendingSlowRecreate("RunBatchAtWidth");
|
||||
if (!ort_session) return; // recreate may have failed; abort safely
|
||||
|
||||
try {
|
||||
// Real number of crops vs. padded batch size submitted to ORT.
|
||||
// Padding to the ladder keeps the recognizer's input-shape set
|
||||
@@ -189,10 +379,22 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
*memory_info_handler, batchInput.data(), batchInput.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: serialize DML inference once per-process
|
||||
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const double runMs =
|
||||
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
||||
RecordRunLatency("RunBatchAtWidth", runMs);
|
||||
|
||||
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
@@ -215,11 +417,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
}
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
// ORT will throw if the model doesn't support a batch dimension > 1.
|
||||
// Fall back to per-image inference for this group.
|
||||
// The original recovery path here covered "model doesn't support
|
||||
// batch dim >1" — a one-shot decision: drop to single-image and
|
||||
// try each crop. We keep that, but ALSO add AMD/DML self-heal
|
||||
// on top: if the per-image fallback ALSO throws, we recreate the
|
||||
// session (same EP first, then CPU) so the next caller gets a
|
||||
// healthy session. The current call's results stay empty for
|
||||
// any crops we couldn't decode — caller tolerates this.
|
||||
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
|
||||
<< bucketW << " (" << e.what()
|
||||
<< ") — falling back to single-image path." << std::endl;
|
||||
bool anyPerImageFailed = false;
|
||||
std::string lastPerImageErr;
|
||||
for (size_t i = 0; i < crops.size(); ++i) {
|
||||
// Direct call (we already hold _mutex via the public RecognizeBatch
|
||||
// wrapper). Replicate the single-image preprocessing here to avoid
|
||||
@@ -232,10 +441,18 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
|
||||
*memory_info_handler, inputData.data(), inputData.size(),
|
||||
inputShape.data(), inputShape.size());
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
std::vector<Ort::Value> outputTensors;
|
||||
{
|
||||
// Option A: per-image fallback path also goes
|
||||
// through the DML serialization gate.
|
||||
ANSCENTER::DmlInferenceGuard dmlGuard(
|
||||
m_engineType,
|
||||
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
|
||||
outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
input_node_names.data(), &inputTensor, 1,
|
||||
output_node_names.data(), num_outputs);
|
||||
}
|
||||
float* outData = outputTensors[0].GetTensorMutableData<float>();
|
||||
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
||||
int seqLen = static_cast<int>(outShape[1]);
|
||||
@@ -245,6 +462,20 @@ void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
||||
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
|
||||
<< e2.what() << std::endl;
|
||||
out[origIndices[i]] = {};
|
||||
anyPerImageFailed = true;
|
||||
lastPerImageErr = e2.what();
|
||||
}
|
||||
}
|
||||
// If even the per-image fallback path is throwing, the session is
|
||||
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
|
||||
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
|
||||
// session (same EP first, CPU as last resort).
|
||||
if (anyPerImageFailed) {
|
||||
try {
|
||||
std::runtime_error proxy(lastPerImageErr);
|
||||
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
|
||||
} catch (...) {
|
||||
// HandleRunFailureAndRecreate already logs; swallow.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -323,10 +554,13 @@ void ONNXOCRRecognizer::Warmup() {
|
||||
// plates pay a one-time spike on first occurrence of batch=16/24.
|
||||
auto totalT0 = std::chrono::high_resolution_clock::now();
|
||||
int warmedShapes = 0;
|
||||
int totalShapes = 0;
|
||||
std::string lastWarmupErr;
|
||||
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
|
||||
const int batchN = kRecWarmupBatchLadder[bs];
|
||||
for (int b = 0; b < kRecNumBuckets; ++b) {
|
||||
const int bucketW = kRecBucketWidths[b];
|
||||
++totalShapes;
|
||||
try {
|
||||
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
|
||||
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
|
||||
@@ -361,6 +595,7 @@ void ONNXOCRRecognizer::Warmup() {
|
||||
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
|
||||
<< batchN << " bucketW=" << bucketW << ": "
|
||||
<< e.what() << std::endl;
|
||||
lastWarmupErr = e.what();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -368,6 +603,30 @@ void ONNXOCRRecognizer::Warmup() {
|
||||
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
|
||||
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
|
||||
<< " shapes in " << totalMs << " ms total" << std::endl;
|
||||
|
||||
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
|
||||
// production traffic will hit Run() failures on every call. Recreate
|
||||
// it now (same EP first, CPU as fallback) so the pipeline can still
|
||||
// serve. We deliberately keep _warmedUp = false in this path; the
|
||||
// next RecognizeBatch will re-trigger Warmup on the recovered session.
|
||||
//
|
||||
// We do NOT trigger self-heal on partial failure — losing 2 of 16
|
||||
// shapes is normal (e.g. some bucket dims aren't supported by all
|
||||
// EPs); the per-shape catch already guards against that.
|
||||
if (warmedShapes == 0 && totalShapes > 0) {
|
||||
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
|
||||
<< totalShapes << " shapes failed — triggering self-heal\n";
|
||||
ANS_DBG("ANSONNXRec_run",
|
||||
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
|
||||
totalShapes, lastWarmupErr.c_str());
|
||||
try {
|
||||
std::runtime_error proxy(lastWarmupErr);
|
||||
HandleRunFailureAndRecreate("Warmup", proxy);
|
||||
} catch (...) {}
|
||||
// Don't set _warmedUp = true; let the recovered session warm up
|
||||
// itself on the next caller's invocation.
|
||||
return;
|
||||
}
|
||||
_warmedUp = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
@@ -51,6 +53,62 @@ private:
|
||||
// CTC greedy decode
|
||||
TextLine CTCDecode(const float* outputData, int seqLen, int numClasses);
|
||||
|
||||
// AMD/DirectML self-heal helper. Called by Recognize / RunBatchAtWidth /
|
||||
// Warmup when ort_session->Run() throws an Ort::Exception. Tries to
|
||||
// recreate the session on the same EP first (transient driver hiccup
|
||||
// recovery — the common case for DML 0x80004005 E_FAIL); if that fails,
|
||||
// falls back to CPU EP. On full failure leaves ort_session in whatever
|
||||
// state Reinitialize left it (possibly nullptr); the next Recognize
|
||||
// call will short-circuit on the !ort_session check and return empty.
|
||||
// Resets _warmedUp so the new session gets re-warmed on first use.
|
||||
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// AMD/DirectML latency-based self-heal (Path B).
|
||||
//
|
||||
// Observed AMD failure mode: after a transient driver event (e.g.
|
||||
// helmet detector hits DML E_FAIL and CPU-falls-back), DML stays in
|
||||
// a degraded state. OCR Run() calls do NOT throw, but their latency
|
||||
// jumps from 50–200 ms to 1000–7000 ms, often rising over time. The
|
||||
// exception-based self-heal above never fires because there's no
|
||||
// exception. This proactive path observes the latency and forces a
|
||||
// session recreate when a Run() exceeds a sanity threshold — the
|
||||
// fresh session typically returns to baseline latency.
|
||||
//
|
||||
// How it works:
|
||||
// 1. Each public method records its Run() duration.
|
||||
// 2. If duration > kSlowRecreateThresholdMs AND we're past warmup
|
||||
// AND we're past the cooldown window, set _slowRecreatePending.
|
||||
// 3. The NEXT call to a public method checks the flag at entry; if
|
||||
// set, it calls Reinitialize(currentEp) BEFORE its Run() — so
|
||||
// the call that sees the flag uses a fresh session and returns
|
||||
// to fast latency.
|
||||
// 4. After a recreate, _lastRecreateMs starts the cooldown timer.
|
||||
//
|
||||
// Threshold tuning: 2 s catches the AMD-degraded individual Run()
|
||||
// pattern observed in production (1–2 s per OCR Run() when DML is
|
||||
// sick). Healthy individual Run() is 50–500 ms, so 2 s leaves
|
||||
// comfortable headroom. First-shape compiles on OpenVINO can hit
|
||||
// ~1.7 s but those happen during Warmup, which is gated out below
|
||||
// via the _warmedUp check. 60 s cooldown prevents thrash if the
|
||||
// driver is in a permanently bad state — at most one recreate per
|
||||
// minute per session.
|
||||
// ----------------------------------------------------------------
|
||||
static constexpr double kSlowRecreateThresholdMs = 2000.0;
|
||||
static constexpr int64_t kRecreateCooldownMs = 60000;
|
||||
|
||||
// Note: a slow Run() detected by RecordRunLatency arms this flag;
|
||||
// ApplyPendingSlowRecreate (called at every public-method entry)
|
||||
// performs the actual Reinitialize and clears the flag. std::atomic
|
||||
// is used so the read at entry is lock-free; the write happens under
|
||||
// the same _mutex the public methods already hold, so the only race
|
||||
// we're guarding is "another public method picked up the flag first."
|
||||
std::atomic<bool> _slowRecreatePending{false};
|
||||
std::chrono::steady_clock::time_point _lastRecreateAt{};
|
||||
|
||||
void RecordRunLatency(const char* context, double ms);
|
||||
void ApplyPendingSlowRecreate(const char* context);
|
||||
|
||||
std::vector<std::string> keys_;
|
||||
int imgH_ = kRecImgH;
|
||||
int imgMaxW_ = kRecImgMaxW;
|
||||
|
||||
Reference in New Issue
Block a user