674 lines
31 KiB
C++
674 lines
31 KiB
C++
#include "ONNXOCRRecognizer.h"
|
|
#include "ANSLicense.h" // ANS_DBG
|
|
|
|
#include <opencv2/imgproc.hpp>
|
|
#include <iostream>
|
|
#include <algorithm>
|
|
#include <numeric>
|
|
#include <cmath>
|
|
#include <cfloat>
|
|
#include <cstring>
|
|
#include <chrono>
|
|
|
|
namespace ANSCENTER {
|
|
namespace onnxocr {
|
|
|
|
ONNXOCRRecognizer::ONNXOCRRecognizer(const std::string& onnx_path, unsigned int num_threads)
|
|
: BasicOrtHandler(onnx_path, num_threads) {
|
|
}
|
|
|
|
ONNXOCRRecognizer::ONNXOCRRecognizer(const std::string& onnx_path,
|
|
const OrtHandlerOptions& options,
|
|
unsigned int num_threads)
|
|
: BasicOrtHandler(onnx_path, options, num_threads) {
|
|
}
|
|
|
|
bool ONNXOCRRecognizer::LoadDictionary(const std::string& dictPath) {
|
|
keys_ = LoadDict(dictPath);
|
|
if (keys_.size() < 2) {
|
|
std::cerr << "[ONNXOCRRecognizer] Failed to load dictionary: " << dictPath << std::endl;
|
|
return false;
|
|
}
|
|
std::cout << "[ONNXOCRRecognizer] Loaded dictionary with " << keys_.size()
|
|
<< " characters from: " << dictPath << std::endl;
|
|
return true;
|
|
}
|
|
|
|
Ort::Value ONNXOCRRecognizer::transform(const cv::Mat& mat) {
|
|
// Not used directly - recognition uses custom preprocess with dynamic width
|
|
cv::Mat resized = ResizeRecImage(mat, imgH_, imgMaxW_);
|
|
resized.convertTo(resized, CV_32FC3);
|
|
auto data = NormalizeAndPermuteCls(resized);
|
|
|
|
input_values_handler.assign(data.begin(), data.end());
|
|
return Ort::Value::CreateTensor<float>(
|
|
*memory_info_handler, input_values_handler.data(), input_values_handler.size(),
|
|
input_node_dims.data(), input_node_dims.size());
|
|
}
|
|
|
|
Ort::Value ONNXOCRRecognizer::transformBatch(const std::vector<cv::Mat>& images) {
|
|
// Not used - recognizer processes single images with dynamic widths
|
|
if (!images.empty()) {
|
|
return transform(images[0]);
|
|
}
|
|
return Ort::Value(nullptr);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Width buckets — every recognizer input is padded up to one of these widths
|
|
// before reaching ORT. This bounds the number of distinct shapes cuDNN ever
|
|
// sees to four, so its HEURISTIC algorithm cache hits on every subsequent
|
|
// call instead of re-tuning per plate. Buckets cover the realistic range:
|
|
// 320 px → short Latin/Japanese plates (most common)
|
|
// 480 px → wider Latin plates with two rows of text
|
|
// 640 px → long single-row plates / multi-line stacked text
|
|
// 960 px → safety upper bound (== kRecImgMaxW)
|
|
// ----------------------------------------------------------------------------
|
|
static constexpr int kRecBucketWidths[] = { 320, 480, 640, 960 };
|
|
static constexpr int kRecNumBuckets = sizeof(kRecBucketWidths) / sizeof(kRecBucketWidths[0]);
|
|
|
|
int ONNXOCRRecognizer::RoundUpToBucket(int resizedW) const {
|
|
const int capped = std::min(resizedW, imgMaxW_);
|
|
for (int b = 0; b < kRecNumBuckets; ++b) {
|
|
if (kRecBucketWidths[b] >= capped) return kRecBucketWidths[b];
|
|
}
|
|
return imgMaxW_;
|
|
}
|
|
|
|
// Round real batch size UP to the next entry in kRecBatchLadder, clamped to
|
|
// kRecMaxBatch. Padded slots in the input tensor are zero-filled and the
|
|
// corresponding output rows are discarded after Run(). See the comment on
|
|
// kRecBatchLadder in ONNXOCRTypes.h for the rationale.
|
|
static int RoundUpToBatchLadder(size_t n) {
|
|
for (int b : kRecBatchLadder) {
|
|
if (b >= static_cast<int>(n)) return b;
|
|
}
|
|
return kRecMaxBatch;
|
|
}
|
|
|
|
// Resize + normalize a single crop into a CHW float vector at width
|
|
// `bucketW`, padding with zeros on the right when needed. The returned
|
|
// vector has exactly 3*imgH_*bucketW elements.
|
|
static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
|
|
int imgH, int bucketW) {
|
|
cv::Mat resized = ResizeRecImage(crop, imgH, bucketW);
|
|
int resizedW = resized.cols;
|
|
resized.convertTo(resized, CV_32FC3);
|
|
auto normalizedData = NormalizeAndPermuteCls(resized);
|
|
|
|
if (resizedW == bucketW) {
|
|
return normalizedData;
|
|
}
|
|
|
|
// Zero-pad on the right (CHW layout)
|
|
std::vector<float> padded(3 * imgH * bucketW, 0.0f);
|
|
for (int c = 0; c < 3; c++) {
|
|
for (int y = 0; y < imgH; y++) {
|
|
std::memcpy(
|
|
&padded[c * imgH * bucketW + y * bucketW],
|
|
&normalizedData[c * imgH * resizedW + y * resizedW],
|
|
resizedW * sizeof(float));
|
|
}
|
|
}
|
|
return padded;
|
|
}
|
|
|
|
// AMD/DirectML latency-based recreate (Path B) — record helper.
|
|
//
|
|
// Called after every successful production-path Run(). If the call
|
|
// took longer than kSlowRecreateThresholdMs (5 s default), and the
|
|
// session has finished its warmup, and we're outside the cooldown
|
|
// window, arm _slowRecreatePending so the NEXT public method will
|
|
// perform a session swap before its own Run().
|
|
//
|
|
// Doesn't actually do the swap here — that has to happen at a clean
|
|
// boundary (no in-flight tensors, no held output Ort::Values), which
|
|
// is precisely the start of the next public method. Calling
|
|
// Reinitialize from inside the middle of an inference path while
|
|
// holding output_node_names / inputTensor / outputTensors references
|
|
// would invalidate them under our feet.
|
|
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
|
|
if (ms <= kSlowRecreateThresholdMs) return;
|
|
if (!_warmedUp) return; // first-shape compile is legitimately slow
|
|
|
|
// Cooldown gate — if we recreated recently, hold off. Avoids a
|
|
// recreate loop if the underlying issue is permanent (e.g. driver
|
|
// truly broken) rather than a transient hiccup.
|
|
const auto now = std::chrono::steady_clock::now();
|
|
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
now - _lastRecreateAt).count();
|
|
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
|
|
sinceLast < kRecreateCooldownMs) {
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
|
|
context, ms, static_cast<long long>(sinceLast));
|
|
return;
|
|
}
|
|
|
|
_slowRecreatePending.store(true, std::memory_order_release);
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": slow Run (" << ms
|
|
<< " ms > threshold) — flagged for proactive session recreate\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
|
|
context, ms, kSlowRecreateThresholdMs);
|
|
}
|
|
|
|
// AMD/DirectML latency-based recreate (Path B) — apply helper.
|
|
//
|
|
// Called at the entry of every public method (Recognize,
|
|
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
|
|
// armed by a previous call's RecordRunLatency, do the session swap
|
|
// here — before any of this call's tensors are constructed. Caller
|
|
// already holds _mutex, so the new ort_session is exclusively ours
|
|
// for the rest of this call.
|
|
//
|
|
// On Reinitialize failure we leave _slowRecreatePending=false so we
|
|
// don't loop forever; the existing exception-based HandleRunFailure-
|
|
// AndRecreate path will fall back to CPU on the next actual throw.
|
|
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
|
|
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
|
|
return;
|
|
}
|
|
const EngineType currentEp = getEngineType();
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
|
|
<< " (latency-triggered)\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: proactive recreate on EP=%d (latency-triggered)",
|
|
context, static_cast<int>(currentEp));
|
|
try {
|
|
Reinitialize(currentEp);
|
|
_warmedUp = false;
|
|
_lastRecreateAt = std::chrono::steady_clock::now();
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": proactive recreate succeeded; next call retries on fresh session\n";
|
|
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
|
|
} catch (const std::exception& e) {
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": proactive recreate failed: " << e.what()
|
|
<< " — keeping existing session, exception path will catch real failures\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: proactive recreate failed (%s)", context, e.what());
|
|
} catch (...) {
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": proactive recreate failed (unknown)\n";
|
|
}
|
|
}
|
|
|
|
// AMD/DirectML self-heal helper. See header comment for detailed contract.
|
|
// In short: on Run() failure, recreate the session on the same EP first
|
|
// (transient driver hiccup recovery — the common case); on a second
|
|
// failure, fall back to CPU EP. Either path leaves ort_session in a
|
|
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
|
|
// result is already lost — caller returns empty.
|
|
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
|
|
const std::exception& e) {
|
|
const EngineType currentEp = getEngineType();
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
|
|
<< ": " << e.what() << " — recreating session\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: Run failed EP=%d msg=%.200s — recreating",
|
|
context, static_cast<int>(currentEp), e.what());
|
|
|
|
// Option A: a Run() exception on a DML session is the strongest
|
|
// signal the AMD driver can't sustain parallel inference. Trip
|
|
// process-wide DML serial mode so future inference calls (from
|
|
// any module) take the global mutex. Idempotent — only the
|
|
// first caller per process flips the flag. No-op on
|
|
// Intel/NVIDIA/CPU.
|
|
if (currentEp == EngineType::AMD_GPU) {
|
|
ANSCENTER::DmlSerialState::TripFromException(
|
|
context, e.what());
|
|
}
|
|
|
|
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
|
|
// are transient driver hiccups; a fresh session typically works. This
|
|
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
|
|
// throw from Run() under normal load, so this path is rarely taken.
|
|
if (currentEp != EngineType::CPU) {
|
|
try {
|
|
Reinitialize(currentEp);
|
|
_warmedUp = false;
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": session recreated on EP=" << static_cast<int>(currentEp)
|
|
<< " — next call will retry on GPU\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: session recreated on EP=%d (next call retries on GPU)",
|
|
context, static_cast<int>(currentEp));
|
|
return;
|
|
} catch (const std::exception& reCreateErr) {
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": same-EP recreate failed: " << reCreateErr.what()
|
|
<< " — falling back to CPU\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: same-EP recreate failed (%s); falling back to CPU",
|
|
context, reCreateErr.what());
|
|
} catch (...) {
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
|
|
}
|
|
}
|
|
|
|
// Step 2: last resort — recreate on CPU. Slower but always works
|
|
// when the model itself is valid. Caller's next call uses CPU EP.
|
|
try {
|
|
Reinitialize(EngineType::CPU);
|
|
_warmedUp = false;
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": fallback session created on CPU\n";
|
|
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
|
|
} catch (const std::exception& cpuErr) {
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": CPU fallback ALSO failed: " << cpuErr.what()
|
|
<< " — ort_session likely null, future calls will return empty\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"%s: CPU fallback failed (%s); ort_session may be null",
|
|
context, cpuErr.what());
|
|
} catch (...) {
|
|
std::cerr << "[ONNXOCRRecognizer] " << context
|
|
<< ": CPU fallback ALSO failed (unknown exception)\n";
|
|
}
|
|
}
|
|
|
|
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
|
|
std::lock_guard<std::mutex> lock(_mutex);
|
|
|
|
if (!ort_session || croppedImage.empty() || keys_.empty()) {
|
|
return {};
|
|
}
|
|
|
|
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
|
|
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
|
|
// are built — so the rest of this method runs on a fresh session.
|
|
ApplyPendingSlowRecreate("Recognize");
|
|
if (!ort_session) return {}; // recreate could have failed; bail safely
|
|
|
|
try {
|
|
// Step 1: aspect-preserving resize to height=imgH_, width capped
|
|
// at imgMaxW_. Then round resized width up to the next bucket.
|
|
cv::Mat resized = ResizeRecImage(croppedImage, imgH_, imgMaxW_);
|
|
const int bucketW = RoundUpToBucket(resized.cols);
|
|
|
|
std::vector<float> inputData = PreprocessCropToBucket(croppedImage, imgH_, bucketW);
|
|
|
|
std::array<int64_t, 4> inputShape = { 1, 3, imgH_, bucketW };
|
|
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
|
|
*memory_info_handler, inputData.data(), inputData.size(),
|
|
inputShape.data(), inputShape.size());
|
|
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::vector<Ort::Value> outputTensors;
|
|
{
|
|
// Option A: serialize DML inference once per-process
|
|
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
|
ANSCENTER::DmlInferenceGuard dmlGuard(
|
|
m_engineType, "ONNXOCRRecognizer::Recognize");
|
|
outputTensors = ort_session->Run(
|
|
Ort::RunOptions{ nullptr },
|
|
input_node_names.data(), &inputTensor, 1,
|
|
output_node_names.data(), num_outputs);
|
|
}
|
|
const auto t1 = std::chrono::steady_clock::now();
|
|
const double runMs =
|
|
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
|
// Latency-based recreate (Path B): arms the flag for the NEXT call
|
|
// if this one was abnormally slow.
|
|
RecordRunLatency("Recognize", runMs);
|
|
|
|
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
|
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
|
|
|
int seqLen = static_cast<int>(outputShape[1]);
|
|
int numClasses = static_cast<int>(outputShape[2]);
|
|
|
|
return CTCDecode(outputData, seqLen, numClasses);
|
|
}
|
|
catch (const Ort::Exception& e) {
|
|
// AMD/DML self-heal: recreate session for next call (same EP first,
|
|
// CPU fallback as last resort). This call's output is lost — return
|
|
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
|
|
// tracker migrates plate text across frames.
|
|
HandleRunFailureAndRecreate("Recognize", e);
|
|
return {};
|
|
}
|
|
}
|
|
|
|
void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
|
|
const std::vector<size_t>& origIndices,
|
|
int bucketW,
|
|
std::vector<TextLine>& out) {
|
|
if (crops.empty()) return;
|
|
|
|
// Path B (AMD/DML): apply pending proactive recreate before any tensors
|
|
// are constructed for this batch. RecognizeBatch holds _mutex around
|
|
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
|
|
// same lock the caller already owns.
|
|
ApplyPendingSlowRecreate("RunBatchAtWidth");
|
|
if (!ort_session) return; // recreate may have failed; abort safely
|
|
|
|
try {
|
|
// Real number of crops vs. padded batch size submitted to ORT.
|
|
// Padding to the ladder keeps the recognizer's input-shape set
|
|
// bounded (≤24 distinct shapes) so OpenVINO/DML don't recompile
|
|
// kernels mid-stream — see kRecBatchLadder in ONNXOCRTypes.h.
|
|
// The padded slots are zeros; their decoded output is discarded.
|
|
const size_t realN = crops.size();
|
|
const size_t paddedN = static_cast<size_t>(RoundUpToBatchLadder(realN));
|
|
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"RunBatchAtWidth: bucketW=%d realN=%zu paddedN=%zu",
|
|
bucketW, realN, paddedN);
|
|
|
|
// Stack realN preprocessed crops into a [paddedN,3,H,W] buffer.
|
|
// The remaining (paddedN - realN) slots stay zero-initialized.
|
|
std::vector<float> batchInput(paddedN * perImage, 0.0f);
|
|
for (size_t i = 0; i < realN; ++i) {
|
|
auto img = PreprocessCropToBucket(crops[i], imgH_, bucketW);
|
|
std::memcpy(&batchInput[i * perImage], img.data(),
|
|
perImage * sizeof(float));
|
|
}
|
|
|
|
std::array<int64_t, 4> inputShape = {
|
|
static_cast<int64_t>(paddedN), 3,
|
|
static_cast<int64_t>(imgH_),
|
|
static_cast<int64_t>(bucketW)
|
|
};
|
|
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
|
|
*memory_info_handler, batchInput.data(), batchInput.size(),
|
|
inputShape.data(), inputShape.size());
|
|
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::vector<Ort::Value> outputTensors;
|
|
{
|
|
// Option A: serialize DML inference once per-process
|
|
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
|
|
ANSCENTER::DmlInferenceGuard dmlGuard(
|
|
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
|
|
outputTensors = ort_session->Run(
|
|
Ort::RunOptions{ nullptr },
|
|
input_node_names.data(), &inputTensor, 1,
|
|
output_node_names.data(), num_outputs);
|
|
}
|
|
const auto t1 = std::chrono::steady_clock::now();
|
|
const double runMs =
|
|
std::chrono::duration<double, std::milli>(t1 - t0).count();
|
|
RecordRunLatency("RunBatchAtWidth", runMs);
|
|
|
|
float* outputData = outputTensors[0].GetTensorMutableData<float>();
|
|
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
|
|
|
// Expected output: [paddedN, seqLen, numClasses]. We only decode
|
|
// the first realN rows; rows [realN..paddedN) are zero-pad noise.
|
|
if (outputShape.size() < 3) {
|
|
std::cerr << "[ONNXOCRRecognizer] Unexpected batch output rank: "
|
|
<< outputShape.size() << std::endl;
|
|
return;
|
|
}
|
|
const int outBatch = static_cast<int>(outputShape[0]);
|
|
const int seqLen = static_cast<int>(outputShape[1]);
|
|
const int numClasses = static_cast<int>(outputShape[2]);
|
|
const size_t perRow = static_cast<size_t>(seqLen) * numClasses;
|
|
|
|
for (size_t i = 0; i < realN && static_cast<int>(i) < outBatch; ++i) {
|
|
TextLine tl = CTCDecode(outputData + i * perRow, seqLen, numClasses);
|
|
out[origIndices[i]] = std::move(tl);
|
|
}
|
|
}
|
|
catch (const Ort::Exception& e) {
|
|
// The original recovery path here covered "model doesn't support
|
|
// batch dim >1" — a one-shot decision: drop to single-image and
|
|
// try each crop. We keep that, but ALSO add AMD/DML self-heal
|
|
// on top: if the per-image fallback ALSO throws, we recreate the
|
|
// session (same EP first, then CPU) so the next caller gets a
|
|
// healthy session. The current call's results stay empty for
|
|
// any crops we couldn't decode — caller tolerates this.
|
|
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
|
|
<< bucketW << " (" << e.what()
|
|
<< ") — falling back to single-image path." << std::endl;
|
|
bool anyPerImageFailed = false;
|
|
std::string lastPerImageErr;
|
|
for (size_t i = 0; i < crops.size(); ++i) {
|
|
// Direct call (we already hold _mutex via the public RecognizeBatch
|
|
// wrapper). Replicate the single-image preprocessing here to avoid
|
|
// re-entering Recognize() and double-locking the mutex.
|
|
try {
|
|
cv::Mat resized = ResizeRecImage(crops[i], imgH_, imgMaxW_);
|
|
int singleBucket = RoundUpToBucket(resized.cols);
|
|
auto inputData = PreprocessCropToBucket(crops[i], imgH_, singleBucket);
|
|
std::array<int64_t, 4> inputShape = { 1, 3, imgH_, singleBucket };
|
|
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
|
|
*memory_info_handler, inputData.data(), inputData.size(),
|
|
inputShape.data(), inputShape.size());
|
|
std::vector<Ort::Value> outputTensors;
|
|
{
|
|
// Option A: per-image fallback path also goes
|
|
// through the DML serialization gate.
|
|
ANSCENTER::DmlInferenceGuard dmlGuard(
|
|
m_engineType,
|
|
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
|
|
outputTensors = ort_session->Run(
|
|
Ort::RunOptions{ nullptr },
|
|
input_node_names.data(), &inputTensor, 1,
|
|
output_node_names.data(), num_outputs);
|
|
}
|
|
float* outData = outputTensors[0].GetTensorMutableData<float>();
|
|
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
|
|
int seqLen = static_cast<int>(outShape[1]);
|
|
int numClasses = static_cast<int>(outShape[2]);
|
|
out[origIndices[i]] = CTCDecode(outData, seqLen, numClasses);
|
|
} catch (const Ort::Exception& e2) {
|
|
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
|
|
<< e2.what() << std::endl;
|
|
out[origIndices[i]] = {};
|
|
anyPerImageFailed = true;
|
|
lastPerImageErr = e2.what();
|
|
}
|
|
}
|
|
// If even the per-image fallback path is throwing, the session is
|
|
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
|
|
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
|
|
// session (same EP first, CPU as last resort).
|
|
if (anyPerImageFailed) {
|
|
try {
|
|
std::runtime_error proxy(lastPerImageErr);
|
|
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
|
|
} catch (...) {
|
|
// HandleRunFailureAndRecreate already logs; swallow.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<TextLine> ONNXOCRRecognizer::RecognizeBatch(const std::vector<cv::Mat>& croppedImages) {
|
|
std::lock_guard<std::mutex> lock(_mutex);
|
|
|
|
std::vector<TextLine> results(croppedImages.size());
|
|
if (!ort_session || croppedImages.empty() || keys_.empty()) {
|
|
return results;
|
|
}
|
|
|
|
// Group crops by their target bucket width
|
|
std::vector<std::vector<cv::Mat>> groupCrops(kRecNumBuckets);
|
|
std::vector<std::vector<size_t>> groupIdx(kRecNumBuckets);
|
|
|
|
for (size_t i = 0; i < croppedImages.size(); ++i) {
|
|
if (croppedImages[i].empty()) continue;
|
|
cv::Mat resized = ResizeRecImage(croppedImages[i], imgH_, imgMaxW_);
|
|
const int bw = RoundUpToBucket(resized.cols);
|
|
// Find bucket index
|
|
int bucketIdx = kRecNumBuckets - 1;
|
|
for (int b = 0; b < kRecNumBuckets; ++b) {
|
|
if (kRecBucketWidths[b] == bw) { bucketIdx = b; break; }
|
|
}
|
|
groupCrops[bucketIdx].push_back(croppedImages[i]);
|
|
groupIdx[bucketIdx].push_back(i);
|
|
}
|
|
|
|
// Run batched inference per non-empty bucket, slicing each bucket
|
|
// group into chunks of at most kRecMaxBatch crops so we never exceed
|
|
// the TRT dynamic profile's max-batch dimension. On a busy scene with
|
|
// (say) 30 plates all falling in bucket 320, we issue two back-to-back
|
|
// batched calls of 24 + 6 instead of one oversized call that would
|
|
// throw "does not satisfy any optimization profiles" and fall off
|
|
// the fast path to the per-image fallback.
|
|
for (int b = 0; b < kRecNumBuckets; ++b) {
|
|
const auto& bucketCrops = groupCrops[b];
|
|
const auto& bucketIndices = groupIdx[b];
|
|
if (bucketCrops.empty()) continue;
|
|
|
|
const int bucketW = kRecBucketWidths[b];
|
|
const size_t total = bucketCrops.size();
|
|
|
|
for (size_t start = 0; start < total; start += kRecMaxBatch) {
|
|
const size_t end = std::min(start + static_cast<size_t>(kRecMaxBatch), total);
|
|
std::vector<cv::Mat> chunkCrops(bucketCrops.begin() + start,
|
|
bucketCrops.begin() + end);
|
|
std::vector<size_t> chunkIdx(bucketIndices.begin() + start,
|
|
bucketIndices.begin() + end);
|
|
RunBatchAtWidth(chunkCrops, chunkIdx, bucketW, results);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
void ONNXOCRRecognizer::Warmup() {
|
|
std::lock_guard<std::mutex> lock(_mutex);
|
|
if (_warmedUp || !ort_session) return;
|
|
|
|
// Dummy 3-channel image, mid-grey, large enough to resize to imgH_
|
|
cv::Mat dummy(imgH_ * 2, kRecBucketWidths[kRecNumBuckets - 1] * 2,
|
|
CV_8UC3, cv::Scalar(128, 128, 128));
|
|
|
|
// Pre-compile every (batchN, bucketW) shape we expect to hit on the
|
|
// hot path. Each unique shape costs ~1.7 s to compile on OpenVINO and
|
|
// ~150 ms on DirectML; without this prewarm, that compile cost shows
|
|
// up as a multi-hundred-ms frame spike the first time a given batch
|
|
// count + plate-width combination occurs in the video stream.
|
|
//
|
|
// We deliberately warm only kRecWarmupBatchLadder ({1,2,4,8}) instead
|
|
// of the full kRecBatchLadder ({1,2,4,8,16,24}) — covers >95% of ALPR
|
|
// frames while keeping engine-init time bounded. Frames with 9+
|
|
// plates pay a one-time spike on first occurrence of batch=16/24.
|
|
auto totalT0 = std::chrono::high_resolution_clock::now();
|
|
int warmedShapes = 0;
|
|
int totalShapes = 0;
|
|
std::string lastWarmupErr;
|
|
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
|
|
const int batchN = kRecWarmupBatchLadder[bs];
|
|
for (int b = 0; b < kRecNumBuckets; ++b) {
|
|
const int bucketW = kRecBucketWidths[b];
|
|
++totalShapes;
|
|
try {
|
|
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
|
|
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
|
|
auto perCrop = PreprocessCropToBucket(dummy, imgH_, bucketW);
|
|
for (int i = 0; i < batchN; ++i) {
|
|
std::memcpy(&batchInput[static_cast<size_t>(i) * perImage],
|
|
perCrop.data(),
|
|
perImage * sizeof(float));
|
|
}
|
|
std::array<int64_t, 4> inputShape = {
|
|
static_cast<int64_t>(batchN), 3,
|
|
static_cast<int64_t>(imgH_),
|
|
static_cast<int64_t>(bucketW)
|
|
};
|
|
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
|
|
*memory_info_handler, batchInput.data(), batchInput.size(),
|
|
inputShape.data(), inputShape.size());
|
|
|
|
auto t0 = std::chrono::high_resolution_clock::now();
|
|
(void)ort_session->Run(
|
|
Ort::RunOptions{ nullptr },
|
|
input_node_names.data(), &inputTensor, 1,
|
|
output_node_names.data(), num_outputs);
|
|
auto t1 = std::chrono::high_resolution_clock::now();
|
|
double ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
|
|
std::cout << "[ONNXOCRRecognizer] Warmup batch=" << batchN
|
|
<< " bucketW=" << bucketW
|
|
<< " " << ms << " ms" << std::endl;
|
|
++warmedShapes;
|
|
}
|
|
catch (const Ort::Exception& e) {
|
|
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
|
|
<< batchN << " bucketW=" << bucketW << ": "
|
|
<< e.what() << std::endl;
|
|
lastWarmupErr = e.what();
|
|
}
|
|
}
|
|
}
|
|
auto totalT1 = std::chrono::high_resolution_clock::now();
|
|
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
|
|
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
|
|
<< " shapes in " << totalMs << " ms total" << std::endl;
|
|
|
|
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
|
|
// production traffic will hit Run() failures on every call. Recreate
|
|
// it now (same EP first, CPU as fallback) so the pipeline can still
|
|
// serve. We deliberately keep _warmedUp = false in this path; the
|
|
// next RecognizeBatch will re-trigger Warmup on the recovered session.
|
|
//
|
|
// We do NOT trigger self-heal on partial failure — losing 2 of 16
|
|
// shapes is normal (e.g. some bucket dims aren't supported by all
|
|
// EPs); the per-shape catch already guards against that.
|
|
if (warmedShapes == 0 && totalShapes > 0) {
|
|
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
|
|
<< totalShapes << " shapes failed — triggering self-heal\n";
|
|
ANS_DBG("ANSONNXRec_run",
|
|
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
|
|
totalShapes, lastWarmupErr.c_str());
|
|
try {
|
|
std::runtime_error proxy(lastWarmupErr);
|
|
HandleRunFailureAndRecreate("Warmup", proxy);
|
|
} catch (...) {}
|
|
// Don't set _warmedUp = true; let the recovered session warm up
|
|
// itself on the next caller's invocation.
|
|
return;
|
|
}
|
|
_warmedUp = true;
|
|
}
|
|
|
|
TextLine ONNXOCRRecognizer::CTCDecode(const float* outputData, int seqLen, int numClasses) {
|
|
TextLine result;
|
|
std::string text;
|
|
std::vector<float> scores;
|
|
|
|
int lastIndex = 0; // CTC blank is index 0
|
|
|
|
for (int t = 0; t < seqLen; t++) {
|
|
// Find argmax for this timestep
|
|
int maxIndex = 0;
|
|
float maxValue = -FLT_MAX;
|
|
|
|
const float* timeStep = outputData + t * numClasses;
|
|
for (int c = 0; c < numClasses; c++) {
|
|
if (timeStep[c] > maxValue) {
|
|
maxValue = timeStep[c];
|
|
maxIndex = c;
|
|
}
|
|
}
|
|
|
|
// CTC decode: skip blanks (index 0) and repeated characters
|
|
if (maxIndex != 0 && maxIndex != lastIndex) {
|
|
if (maxIndex > 0 && maxIndex < static_cast<int>(keys_.size())) {
|
|
text += keys_[maxIndex]; // keys_[0]="#"(blank), keys_[1]=first_char, etc.
|
|
// Use raw model output value as confidence (PaddleOCR v5 models include softmax)
|
|
scores.push_back(maxValue);
|
|
}
|
|
}
|
|
lastIndex = maxIndex;
|
|
}
|
|
|
|
result.text = text;
|
|
if (!scores.empty()) {
|
|
result.score = std::accumulate(scores.begin(), scores.end(), 0.0f) /
|
|
static_cast<float>(scores.size());
|
|
}
|
|
return result;
|
|
}
|
|
|
|
} // namespace onnxocr
|
|
} // namespace ANSCENTER
|