Files
ANSCORE/modules/ANSOCR/ANSONNXOCR/ONNXOCRRecognizer.cpp
T

674 lines
31 KiB
C++

#include "ONNXOCRRecognizer.h"
#include "ANSLicense.h" // ANS_DBG
#include <opencv2/imgproc.hpp>
#include <iostream>
#include <algorithm>
#include <numeric>
#include <cmath>
#include <cfloat>
#include <cstring>
#include <chrono>
namespace ANSCENTER {
namespace onnxocr {
ONNXOCRRecognizer::ONNXOCRRecognizer(const std::string& onnx_path, unsigned int num_threads)
: BasicOrtHandler(onnx_path, num_threads) {
}
ONNXOCRRecognizer::ONNXOCRRecognizer(const std::string& onnx_path,
const OrtHandlerOptions& options,
unsigned int num_threads)
: BasicOrtHandler(onnx_path, options, num_threads) {
}
bool ONNXOCRRecognizer::LoadDictionary(const std::string& dictPath) {
keys_ = LoadDict(dictPath);
if (keys_.size() < 2) {
std::cerr << "[ONNXOCRRecognizer] Failed to load dictionary: " << dictPath << std::endl;
return false;
}
std::cout << "[ONNXOCRRecognizer] Loaded dictionary with " << keys_.size()
<< " characters from: " << dictPath << std::endl;
return true;
}
Ort::Value ONNXOCRRecognizer::transform(const cv::Mat& mat) {
// Not used directly - recognition uses custom preprocess with dynamic width
cv::Mat resized = ResizeRecImage(mat, imgH_, imgMaxW_);
resized.convertTo(resized, CV_32FC3);
auto data = NormalizeAndPermuteCls(resized);
input_values_handler.assign(data.begin(), data.end());
return Ort::Value::CreateTensor<float>(
*memory_info_handler, input_values_handler.data(), input_values_handler.size(),
input_node_dims.data(), input_node_dims.size());
}
Ort::Value ONNXOCRRecognizer::transformBatch(const std::vector<cv::Mat>& images) {
// Not used - recognizer processes single images with dynamic widths
if (!images.empty()) {
return transform(images[0]);
}
return Ort::Value(nullptr);
}
// ----------------------------------------------------------------------------
// Width buckets — every recognizer input is padded up to one of these widths
// before reaching ORT. This bounds the number of distinct shapes cuDNN ever
// sees to four, so its HEURISTIC algorithm cache hits on every subsequent
// call instead of re-tuning per plate. Buckets cover the realistic range:
// 320 px → short Latin/Japanese plates (most common)
// 480 px → wider Latin plates with two rows of text
// 640 px → long single-row plates / multi-line stacked text
// 960 px → safety upper bound (== kRecImgMaxW)
// ----------------------------------------------------------------------------
static constexpr int kRecBucketWidths[] = { 320, 480, 640, 960 };
static constexpr int kRecNumBuckets = sizeof(kRecBucketWidths) / sizeof(kRecBucketWidths[0]);
int ONNXOCRRecognizer::RoundUpToBucket(int resizedW) const {
const int capped = std::min(resizedW, imgMaxW_);
for (int b = 0; b < kRecNumBuckets; ++b) {
if (kRecBucketWidths[b] >= capped) return kRecBucketWidths[b];
}
return imgMaxW_;
}
// Round real batch size UP to the next entry in kRecBatchLadder, clamped to
// kRecMaxBatch. Padded slots in the input tensor are zero-filled and the
// corresponding output rows are discarded after Run(). See the comment on
// kRecBatchLadder in ONNXOCRTypes.h for the rationale.
static int RoundUpToBatchLadder(size_t n) {
for (int b : kRecBatchLadder) {
if (b >= static_cast<int>(n)) return b;
}
return kRecMaxBatch;
}
// Resize + normalize a single crop into a CHW float vector at width
// `bucketW`, padding with zeros on the right when needed. The returned
// vector has exactly 3*imgH_*bucketW elements.
static std::vector<float> PreprocessCropToBucket(const cv::Mat& crop,
int imgH, int bucketW) {
cv::Mat resized = ResizeRecImage(crop, imgH, bucketW);
int resizedW = resized.cols;
resized.convertTo(resized, CV_32FC3);
auto normalizedData = NormalizeAndPermuteCls(resized);
if (resizedW == bucketW) {
return normalizedData;
}
// Zero-pad on the right (CHW layout)
std::vector<float> padded(3 * imgH * bucketW, 0.0f);
for (int c = 0; c < 3; c++) {
for (int y = 0; y < imgH; y++) {
std::memcpy(
&padded[c * imgH * bucketW + y * bucketW],
&normalizedData[c * imgH * resizedW + y * resizedW],
resizedW * sizeof(float));
}
}
return padded;
}
// AMD/DirectML latency-based recreate (Path B) — record helper.
//
// Called after every successful production-path Run(). If the call
// took longer than kSlowRecreateThresholdMs (5 s default), and the
// session has finished its warmup, and we're outside the cooldown
// window, arm _slowRecreatePending so the NEXT public method will
// perform a session swap before its own Run().
//
// Doesn't actually do the swap here — that has to happen at a clean
// boundary (no in-flight tensors, no held output Ort::Values), which
// is precisely the start of the next public method. Calling
// Reinitialize from inside the middle of an inference path while
// holding output_node_names / inputTensor / outputTensors references
// would invalidate them under our feet.
void ONNXOCRRecognizer::RecordRunLatency(const char* context, double ms) {
if (ms <= kSlowRecreateThresholdMs) return;
if (!_warmedUp) return; // first-shape compile is legitimately slow
// Cooldown gate — if we recreated recently, hold off. Avoids a
// recreate loop if the underlying issue is permanent (e.g. driver
// truly broken) rather than a transient hiccup.
const auto now = std::chrono::steady_clock::now();
const auto sinceLast = std::chrono::duration_cast<std::chrono::milliseconds>(
now - _lastRecreateAt).count();
if (_lastRecreateAt.time_since_epoch().count() != 0 &&
sinceLast < kRecreateCooldownMs) {
ANS_DBG("ANSONNXRec_run",
"%s: slow Run (%.0f ms) but in cooldown (%lldms since last recreate)",
context, ms, static_cast<long long>(sinceLast));
return;
}
_slowRecreatePending.store(true, std::memory_order_release);
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": slow Run (" << ms
<< " ms > threshold) — flagged for proactive session recreate\n";
ANS_DBG("ANSONNXRec_run",
"%s: slow Run %.0fms > %.0fms — armed proactive recreate",
context, ms, kSlowRecreateThresholdMs);
}
// AMD/DirectML latency-based recreate (Path B) — apply helper.
//
// Called at the entry of every public method (Recognize,
// RunBatchAtWidth, Warmup, RecognizeBatch). If the slow flag was
// armed by a previous call's RecordRunLatency, do the session swap
// here — before any of this call's tensors are constructed. Caller
// already holds _mutex, so the new ort_session is exclusively ours
// for the rest of this call.
//
// On Reinitialize failure we leave _slowRecreatePending=false so we
// don't loop forever; the existing exception-based HandleRunFailure-
// AndRecreate path will fall back to CPU on the next actual throw.
void ONNXOCRRecognizer::ApplyPendingSlowRecreate(const char* context) {
if (!_slowRecreatePending.exchange(false, std::memory_order_acquire)) {
return;
}
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate on EP=" << static_cast<int>(currentEp)
<< " (latency-triggered)\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate on EP=%d (latency-triggered)",
context, static_cast<int>(currentEp));
try {
Reinitialize(currentEp);
_warmedUp = false;
_lastRecreateAt = std::chrono::steady_clock::now();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate succeeded; next call retries on fresh session\n";
ANS_DBG("ANSONNXRec_run", "%s: proactive recreate OK", context);
} catch (const std::exception& e) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed: " << e.what()
<< " — keeping existing session, exception path will catch real failures\n";
ANS_DBG("ANSONNXRec_run",
"%s: proactive recreate failed (%s)", context, e.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": proactive recreate failed (unknown)\n";
}
}
// AMD/DirectML self-heal helper. See header comment for detailed contract.
// In short: on Run() failure, recreate the session on the same EP first
// (transient driver hiccup recovery — the common case); on a second
// failure, fall back to CPU EP. Either path leaves ort_session in a
// usable state for the NEXT Recognize/RunBatchAtWidth call. This call's
// result is already lost — caller returns empty.
void ONNXOCRRecognizer::HandleRunFailureAndRecreate(const char* context,
const std::exception& e) {
const EngineType currentEp = getEngineType();
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": Run() failed on EP=" << static_cast<int>(currentEp)
<< ": " << e.what() << " — recreating session\n";
ANS_DBG("ANSONNXRec_run",
"%s: Run failed EP=%d msg=%.200s — recreating",
context, static_cast<int>(currentEp), e.what());
// Option A: a Run() exception on a DML session is the strongest
// signal the AMD driver can't sustain parallel inference. Trip
// process-wide DML serial mode so future inference calls (from
// any module) take the global mutex. Idempotent — only the
// first caller per process flips the flag. No-op on
// Intel/NVIDIA/CPU.
if (currentEp == EngineType::AMD_GPU) {
ANSCENTER::DmlSerialState::TripFromException(
context, e.what());
}
// Step 1: try to recover on the same EP. Most DML 0x80004005 failures
// are transient driver hiccups; a fresh session typically works. This
// is also a no-op cost on Intel/NVIDIA — those EPs effectively never
// throw from Run() under normal load, so this path is rarely taken.
if (currentEp != EngineType::CPU) {
try {
Reinitialize(currentEp);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": session recreated on EP=" << static_cast<int>(currentEp)
<< " — next call will retry on GPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: session recreated on EP=%d (next call retries on GPU)",
context, static_cast<int>(currentEp));
return;
} catch (const std::exception& reCreateErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed: " << reCreateErr.what()
<< " — falling back to CPU\n";
ANS_DBG("ANSONNXRec_run",
"%s: same-EP recreate failed (%s); falling back to CPU",
context, reCreateErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": same-EP recreate failed (unknown) — falling back to CPU\n";
}
}
// Step 2: last resort — recreate on CPU. Slower but always works
// when the model itself is valid. Caller's next call uses CPU EP.
try {
Reinitialize(EngineType::CPU);
_warmedUp = false;
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": fallback session created on CPU\n";
ANS_DBG("ANSONNXRec_run", "%s: CPU fallback session created", context);
} catch (const std::exception& cpuErr) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed: " << cpuErr.what()
<< " — ort_session likely null, future calls will return empty\n";
ANS_DBG("ANSONNXRec_run",
"%s: CPU fallback failed (%s); ort_session may be null",
context, cpuErr.what());
} catch (...) {
std::cerr << "[ONNXOCRRecognizer] " << context
<< ": CPU fallback ALSO failed (unknown exception)\n";
}
}
TextLine ONNXOCRRecognizer::Recognize(const cv::Mat& croppedImage) {
std::lock_guard<std::mutex> lock(_mutex);
if (!ort_session || croppedImage.empty() || keys_.empty()) {
return {};
}
// Path B (AMD/DML): if a previous call observed a slow Run(), it armed
// a proactive-recreate flag. Apply it now — BEFORE this call's tensors
// are built — so the rest of this method runs on a fresh session.
ApplyPendingSlowRecreate("Recognize");
if (!ort_session) return {}; // recreate could have failed; bail safely
try {
// Step 1: aspect-preserving resize to height=imgH_, width capped
// at imgMaxW_. Then round resized width up to the next bucket.
cv::Mat resized = ResizeRecImage(croppedImage, imgH_, imgMaxW_);
const int bucketW = RoundUpToBucket(resized.cols);
std::vector<float> inputData = PreprocessCropToBucket(croppedImage, imgH_, bucketW);
std::array<int64_t, 4> inputShape = { 1, 3, imgH_, bucketW };
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::Recognize");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
// Latency-based recreate (Path B): arms the flag for the NEXT call
// if this one was abnormally slow.
RecordRunLatency("Recognize", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int seqLen = static_cast<int>(outputShape[1]);
int numClasses = static_cast<int>(outputShape[2]);
return CTCDecode(outputData, seqLen, numClasses);
}
catch (const Ort::Exception& e) {
// AMD/DML self-heal: recreate session for next call (same EP first,
// CPU fallback as last resort). This call's output is lost — return
// empty. The OCR pipeline tolerates dropped frames; the ALPR voting
// tracker migrates plate text across frames.
HandleRunFailureAndRecreate("Recognize", e);
return {};
}
}
void ONNXOCRRecognizer::RunBatchAtWidth(const std::vector<cv::Mat>& crops,
const std::vector<size_t>& origIndices,
int bucketW,
std::vector<TextLine>& out) {
if (crops.empty()) return;
// Path B (AMD/DML): apply pending proactive recreate before any tensors
// are constructed for this batch. RecognizeBatch holds _mutex around
// every RunBatchAtWidth chunk it dispatches, so we're safe under the
// same lock the caller already owns.
ApplyPendingSlowRecreate("RunBatchAtWidth");
if (!ort_session) return; // recreate may have failed; abort safely
try {
// Real number of crops vs. padded batch size submitted to ORT.
// Padding to the ladder keeps the recognizer's input-shape set
// bounded (≤24 distinct shapes) so OpenVINO/DML don't recompile
// kernels mid-stream — see kRecBatchLadder in ONNXOCRTypes.h.
// The padded slots are zeros; their decoded output is discarded.
const size_t realN = crops.size();
const size_t paddedN = static_cast<size_t>(RoundUpToBatchLadder(realN));
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
ANS_DBG("ANSONNXRec_run",
"RunBatchAtWidth: bucketW=%d realN=%zu paddedN=%zu",
bucketW, realN, paddedN);
// Stack realN preprocessed crops into a [paddedN,3,H,W] buffer.
// The remaining (paddedN - realN) slots stay zero-initialized.
std::vector<float> batchInput(paddedN * perImage, 0.0f);
for (size_t i = 0; i < realN; ++i) {
auto img = PreprocessCropToBucket(crops[i], imgH_, bucketW);
std::memcpy(&batchInput[i * perImage], img.data(),
perImage * sizeof(float));
}
std::array<int64_t, 4> inputShape = {
static_cast<int64_t>(paddedN), 3,
static_cast<int64_t>(imgH_),
static_cast<int64_t>(bucketW)
};
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, batchInput.data(), batchInput.size(),
inputShape.data(), inputShape.size());
const auto t0 = std::chrono::steady_clock::now();
std::vector<Ort::Value> outputTensors;
{
// Option A: serialize DML inference once per-process
// serial mode has been tripped (no-op on Intel/NVIDIA/CPU).
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType, "ONNXOCRRecognizer::RunBatchAtWidth");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
const auto t1 = std::chrono::steady_clock::now();
const double runMs =
std::chrono::duration<double, std::milli>(t1 - t0).count();
RecordRunLatency("RunBatchAtWidth", runMs);
float* outputData = outputTensors[0].GetTensorMutableData<float>();
auto outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
// Expected output: [paddedN, seqLen, numClasses]. We only decode
// the first realN rows; rows [realN..paddedN) are zero-pad noise.
if (outputShape.size() < 3) {
std::cerr << "[ONNXOCRRecognizer] Unexpected batch output rank: "
<< outputShape.size() << std::endl;
return;
}
const int outBatch = static_cast<int>(outputShape[0]);
const int seqLen = static_cast<int>(outputShape[1]);
const int numClasses = static_cast<int>(outputShape[2]);
const size_t perRow = static_cast<size_t>(seqLen) * numClasses;
for (size_t i = 0; i < realN && static_cast<int>(i) < outBatch; ++i) {
TextLine tl = CTCDecode(outputData + i * perRow, seqLen, numClasses);
out[origIndices[i]] = std::move(tl);
}
}
catch (const Ort::Exception& e) {
// The original recovery path here covered "model doesn't support
// batch dim >1" — a one-shot decision: drop to single-image and
// try each crop. We keep that, but ALSO add AMD/DML self-heal
// on top: if the per-image fallback ALSO throws, we recreate the
// session (same EP first, then CPU) so the next caller gets a
// healthy session. The current call's results stay empty for
// any crops we couldn't decode — caller tolerates this.
std::cerr << "[ONNXOCRRecognizer] Batch inference failed at bucketW="
<< bucketW << " (" << e.what()
<< ") — falling back to single-image path." << std::endl;
bool anyPerImageFailed = false;
std::string lastPerImageErr;
for (size_t i = 0; i < crops.size(); ++i) {
// Direct call (we already hold _mutex via the public RecognizeBatch
// wrapper). Replicate the single-image preprocessing here to avoid
// re-entering Recognize() and double-locking the mutex.
try {
cv::Mat resized = ResizeRecImage(crops[i], imgH_, imgMaxW_);
int singleBucket = RoundUpToBucket(resized.cols);
auto inputData = PreprocessCropToBucket(crops[i], imgH_, singleBucket);
std::array<int64_t, 4> inputShape = { 1, 3, imgH_, singleBucket };
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, inputData.data(), inputData.size(),
inputShape.data(), inputShape.size());
std::vector<Ort::Value> outputTensors;
{
// Option A: per-image fallback path also goes
// through the DML serialization gate.
ANSCENTER::DmlInferenceGuard dmlGuard(
m_engineType,
"ONNXOCRRecognizer::RunBatchAtWidth/per-image");
outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
}
float* outData = outputTensors[0].GetTensorMutableData<float>();
auto outShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
int seqLen = static_cast<int>(outShape[1]);
int numClasses = static_cast<int>(outShape[2]);
out[origIndices[i]] = CTCDecode(outData, seqLen, numClasses);
} catch (const Ort::Exception& e2) {
std::cerr << "[ONNXOCRRecognizer] Single-image fallback also failed: "
<< e2.what() << std::endl;
out[origIndices[i]] = {};
anyPerImageFailed = true;
lastPerImageErr = e2.what();
}
}
// If even the per-image fallback path is throwing, the session is
// genuinely unhealthy (DML device-lost, not just a batch-shape bug).
// Trigger the AMD/DML self-heal so the NEXT batch lands on a fresh
// session (same EP first, CPU as last resort).
if (anyPerImageFailed) {
try {
std::runtime_error proxy(lastPerImageErr);
HandleRunFailureAndRecreate("RunBatchAtWidth", proxy);
} catch (...) {
// HandleRunFailureAndRecreate already logs; swallow.
}
}
}
}
std::vector<TextLine> ONNXOCRRecognizer::RecognizeBatch(const std::vector<cv::Mat>& croppedImages) {
std::lock_guard<std::mutex> lock(_mutex);
std::vector<TextLine> results(croppedImages.size());
if (!ort_session || croppedImages.empty() || keys_.empty()) {
return results;
}
// Group crops by their target bucket width
std::vector<std::vector<cv::Mat>> groupCrops(kRecNumBuckets);
std::vector<std::vector<size_t>> groupIdx(kRecNumBuckets);
for (size_t i = 0; i < croppedImages.size(); ++i) {
if (croppedImages[i].empty()) continue;
cv::Mat resized = ResizeRecImage(croppedImages[i], imgH_, imgMaxW_);
const int bw = RoundUpToBucket(resized.cols);
// Find bucket index
int bucketIdx = kRecNumBuckets - 1;
for (int b = 0; b < kRecNumBuckets; ++b) {
if (kRecBucketWidths[b] == bw) { bucketIdx = b; break; }
}
groupCrops[bucketIdx].push_back(croppedImages[i]);
groupIdx[bucketIdx].push_back(i);
}
// Run batched inference per non-empty bucket, slicing each bucket
// group into chunks of at most kRecMaxBatch crops so we never exceed
// the TRT dynamic profile's max-batch dimension. On a busy scene with
// (say) 30 plates all falling in bucket 320, we issue two back-to-back
// batched calls of 24 + 6 instead of one oversized call that would
// throw "does not satisfy any optimization profiles" and fall off
// the fast path to the per-image fallback.
for (int b = 0; b < kRecNumBuckets; ++b) {
const auto& bucketCrops = groupCrops[b];
const auto& bucketIndices = groupIdx[b];
if (bucketCrops.empty()) continue;
const int bucketW = kRecBucketWidths[b];
const size_t total = bucketCrops.size();
for (size_t start = 0; start < total; start += kRecMaxBatch) {
const size_t end = std::min(start + static_cast<size_t>(kRecMaxBatch), total);
std::vector<cv::Mat> chunkCrops(bucketCrops.begin() + start,
bucketCrops.begin() + end);
std::vector<size_t> chunkIdx(bucketIndices.begin() + start,
bucketIndices.begin() + end);
RunBatchAtWidth(chunkCrops, chunkIdx, bucketW, results);
}
}
return results;
}
void ONNXOCRRecognizer::Warmup() {
std::lock_guard<std::mutex> lock(_mutex);
if (_warmedUp || !ort_session) return;
// Dummy 3-channel image, mid-grey, large enough to resize to imgH_
cv::Mat dummy(imgH_ * 2, kRecBucketWidths[kRecNumBuckets - 1] * 2,
CV_8UC3, cv::Scalar(128, 128, 128));
// Pre-compile every (batchN, bucketW) shape we expect to hit on the
// hot path. Each unique shape costs ~1.7 s to compile on OpenVINO and
// ~150 ms on DirectML; without this prewarm, that compile cost shows
// up as a multi-hundred-ms frame spike the first time a given batch
// count + plate-width combination occurs in the video stream.
//
// We deliberately warm only kRecWarmupBatchLadder ({1,2,4,8}) instead
// of the full kRecBatchLadder ({1,2,4,8,16,24}) — covers >95% of ALPR
// frames while keeping engine-init time bounded. Frames with 9+
// plates pay a one-time spike on first occurrence of batch=16/24.
auto totalT0 = std::chrono::high_resolution_clock::now();
int warmedShapes = 0;
int totalShapes = 0;
std::string lastWarmupErr;
for (int bs = 0; bs < kRecNumWarmupBatchSizes; ++bs) {
const int batchN = kRecWarmupBatchLadder[bs];
for (int b = 0; b < kRecNumBuckets; ++b) {
const int bucketW = kRecBucketWidths[b];
++totalShapes;
try {
const size_t perImage = static_cast<size_t>(3) * imgH_ * bucketW;
std::vector<float> batchInput(static_cast<size_t>(batchN) * perImage, 0.0f);
auto perCrop = PreprocessCropToBucket(dummy, imgH_, bucketW);
for (int i = 0; i < batchN; ++i) {
std::memcpy(&batchInput[static_cast<size_t>(i) * perImage],
perCrop.data(),
perImage * sizeof(float));
}
std::array<int64_t, 4> inputShape = {
static_cast<int64_t>(batchN), 3,
static_cast<int64_t>(imgH_),
static_cast<int64_t>(bucketW)
};
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
*memory_info_handler, batchInput.data(), batchInput.size(),
inputShape.data(), inputShape.size());
auto t0 = std::chrono::high_resolution_clock::now();
(void)ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(), &inputTensor, 1,
output_node_names.data(), num_outputs);
auto t1 = std::chrono::high_resolution_clock::now();
double ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
std::cout << "[ONNXOCRRecognizer] Warmup batch=" << batchN
<< " bucketW=" << bucketW
<< " " << ms << " ms" << std::endl;
++warmedShapes;
}
catch (const Ort::Exception& e) {
std::cerr << "[ONNXOCRRecognizer] Warmup failed at batch="
<< batchN << " bucketW=" << bucketW << ": "
<< e.what() << std::endl;
lastWarmupErr = e.what();
}
}
}
auto totalT1 = std::chrono::high_resolution_clock::now();
double totalMs = std::chrono::duration<double, std::milli>(totalT1 - totalT0).count();
std::cout << "[ONNXOCRRecognizer] Warmup complete: " << warmedShapes
<< " shapes in " << totalMs << " ms total" << std::endl;
// AMD/DML self-heal: if NO shape warmed up, the session is dead and
// production traffic will hit Run() failures on every call. Recreate
// it now (same EP first, CPU as fallback) so the pipeline can still
// serve. We deliberately keep _warmedUp = false in this path; the
// next RecognizeBatch will re-trigger Warmup on the recovered session.
//
// We do NOT trigger self-heal on partial failure — losing 2 of 16
// shapes is normal (e.g. some bucket dims aren't supported by all
// EPs); the per-shape catch already guards against that.
if (warmedShapes == 0 && totalShapes > 0) {
std::cerr << "[ONNXOCRRecognizer] Warmup: ALL "
<< totalShapes << " shapes failed — triggering self-heal\n";
ANS_DBG("ANSONNXRec_run",
"Warmup: 0/%d shapes succeeded (last err=%.200s); recreating session",
totalShapes, lastWarmupErr.c_str());
try {
std::runtime_error proxy(lastWarmupErr);
HandleRunFailureAndRecreate("Warmup", proxy);
} catch (...) {}
// Don't set _warmedUp = true; let the recovered session warm up
// itself on the next caller's invocation.
return;
}
_warmedUp = true;
}
TextLine ONNXOCRRecognizer::CTCDecode(const float* outputData, int seqLen, int numClasses) {
TextLine result;
std::string text;
std::vector<float> scores;
int lastIndex = 0; // CTC blank is index 0
for (int t = 0; t < seqLen; t++) {
// Find argmax for this timestep
int maxIndex = 0;
float maxValue = -FLT_MAX;
const float* timeStep = outputData + t * numClasses;
for (int c = 0; c < numClasses; c++) {
if (timeStep[c] > maxValue) {
maxValue = timeStep[c];
maxIndex = c;
}
}
// CTC decode: skip blanks (index 0) and repeated characters
if (maxIndex != 0 && maxIndex != lastIndex) {
if (maxIndex > 0 && maxIndex < static_cast<int>(keys_.size())) {
text += keys_[maxIndex]; // keys_[0]="#"(blank), keys_[1]=first_char, etc.
// Use raw model output value as confidence (PaddleOCR v5 models include softmax)
scores.push_back(maxValue);
}
}
lastIndex = maxIndex;
}
result.text = text;
if (!scores.empty()) {
result.score = std::accumulate(scores.begin(), scores.end(), 0.0f) /
static_cast<float>(scores.size());
}
return result;
}
} // namespace onnxocr
} // namespace ANSCENTER