Fix AMD ALPR_OCR and custom model
This commit is contained in:
@@ -5,6 +5,8 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
|
||||
namespace ANSCENTER {
|
||||
namespace onnxocr {
|
||||
@@ -51,6 +53,62 @@ private:
|
||||
// CTC greedy decode
|
||||
TextLine CTCDecode(const float* outputData, int seqLen, int numClasses);
|
||||
|
||||
// AMD/DirectML self-heal helper. Called by Recognize / RunBatchAtWidth /
|
||||
// Warmup when ort_session->Run() throws an Ort::Exception. Tries to
|
||||
// recreate the session on the same EP first (transient driver hiccup
|
||||
// recovery — the common case for DML 0x80004005 E_FAIL); if that fails,
|
||||
// falls back to CPU EP. On full failure leaves ort_session in whatever
|
||||
// state Reinitialize left it (possibly nullptr); the next Recognize
|
||||
// call will short-circuit on the !ort_session check and return empty.
|
||||
// Resets _warmedUp so the new session gets re-warmed on first use.
|
||||
void HandleRunFailureAndRecreate(const char* context, const std::exception& e);
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// AMD/DirectML latency-based self-heal (Path B).
|
||||
//
|
||||
// Observed AMD failure mode: after a transient driver event (e.g.
|
||||
// helmet detector hits DML E_FAIL and CPU-falls-back), DML stays in
|
||||
// a degraded state. OCR Run() calls do NOT throw, but their latency
|
||||
// jumps from 50–200 ms to 1000–7000 ms, often rising over time. The
|
||||
// exception-based self-heal above never fires because there's no
|
||||
// exception. This proactive path observes the latency and forces a
|
||||
// session recreate when a Run() exceeds a sanity threshold — the
|
||||
// fresh session typically returns to baseline latency.
|
||||
//
|
||||
// How it works:
|
||||
// 1. Each public method records its Run() duration.
|
||||
// 2. If duration > kSlowRecreateThresholdMs AND we're past warmup
|
||||
// AND we're past the cooldown window, set _slowRecreatePending.
|
||||
// 3. The NEXT call to a public method checks the flag at entry; if
|
||||
// set, it calls Reinitialize(currentEp) BEFORE its Run() — so
|
||||
// the call that sees the flag uses a fresh session and returns
|
||||
// to fast latency.
|
||||
// 4. After a recreate, _lastRecreateMs starts the cooldown timer.
|
||||
//
|
||||
// Threshold tuning: 2 s catches the AMD-degraded individual Run()
|
||||
// pattern observed in production (1–2 s per OCR Run() when DML is
|
||||
// sick). Healthy individual Run() is 50–500 ms, so 2 s leaves
|
||||
// comfortable headroom. First-shape compiles on OpenVINO can hit
|
||||
// ~1.7 s but those happen during Warmup, which is gated out below
|
||||
// via the _warmedUp check. 60 s cooldown prevents thrash if the
|
||||
// driver is in a permanently bad state — at most one recreate per
|
||||
// minute per session.
|
||||
// ----------------------------------------------------------------
|
||||
static constexpr double kSlowRecreateThresholdMs = 2000.0;
|
||||
static constexpr int64_t kRecreateCooldownMs = 60000;
|
||||
|
||||
// Note: a slow Run() detected by RecordRunLatency arms this flag;
|
||||
// ApplyPendingSlowRecreate (called at every public-method entry)
|
||||
// performs the actual Reinitialize and clears the flag. std::atomic
|
||||
// is used so the read at entry is lock-free; the write happens under
|
||||
// the same _mutex the public methods already hold, so the only race
|
||||
// we're guarding is "another public method picked up the flag first."
|
||||
std::atomic<bool> _slowRecreatePending{false};
|
||||
std::chrono::steady_clock::time_point _lastRecreateAt{};
|
||||
|
||||
void RecordRunLatency(const char* context, double ms);
|
||||
void ApplyPendingSlowRecreate(const char* context);
|
||||
|
||||
std::vector<std::string> keys_;
|
||||
int imgH_ = kRecImgH;
|
||||
int imgMaxW_ = kRecImgMaxW;
|
||||
|
||||
Reference in New Issue
Block a user