Files
ANSCORE/modules/ANSODEngine/ANSONNXYOLO.h
T

353 lines
18 KiB
C++
Raw Normal View History

2026-03-28 16:54:11 +11:00
#ifndef ANSONNXYOLO_H
#define ANSONNXYOLO_H
#pragma once
#include "ANSEngineCommon.h"
#include "ONNXEngine.h"
namespace ANSCENTER {
// ====================================================================
// ONNXYOLO — Ultralytics YOLO inference via ONNX Runtime
//
// Compatible with ALL Ultralytics YOLO tasks and versions
// (v8, v9, v10, v11, v26+) by auto-detecting the task type
// from output tensor shapes at inference time.
//
// Supported tasks:
// - Detection : [B,300,6] end2end or [B, nc+4, N] legacy
// - OBB : [B,300,7] end2end or [B, nc+5, N] legacy
// - Segmentation: [B,300,38]+protos or [B, nc+36, N]+protos
// - Pose : [B,300,6+nk*3] or [B, nc+4+nk*3, N] legacy
// - Classification: [B, nc]
//
// Preprocessing follows the exact Ultralytics LetterBox transform:
// - center=True, stride=32, pad_value=114
// - Ultralytics-compatible -0.1/+0.1 rounding for deterministic padding
// - BGR→RGB, /255.0 normalisation, HWC→CHW
// ====================================================================
class ONNXENGINE_API ONNXYOLO : public BasicOrtHandler
{
public:
explicit ONNXYOLO(const std::string& _onnx_path,
unsigned int _num_threads = 1);
explicit ONNXYOLO(const std::string& _onnx_path,
EngineType engineType,
unsigned int _num_threads = 1);
~ONNXYOLO() override = default;
/// Run inference on a single image.
/// Auto-detects the task type (detect/segment/obb/pose/classify)
/// from the ONNX model's output tensor shapes.
std::vector<Object> detect(const cv::Mat& image,
const std::vector<std::string>& classNames,
float confThreshold = 0.25f,
float iouThreshold = 0.45f,
int numKPS = 0);
/// True after detect() if the last inference was classification.
bool lastWasClassification = false;
/// Run batched inference on multiple images in a single ONNX session call.
/// Falls back to sequential detect() if the model has fixed batch=1.
std::vector<std::vector<Object>> detectBatch(
const std::vector<cv::Mat>& images,
const std::vector<std::string>& classNames,
float confThreshold = 0.25f,
float iouThreshold = 0.45f,
int numKPS = 0);
/// True after detectBatch() if the batch was classification.
bool lastBatchWasClassification = false;
/// Override the input image shape for dynamic-input models.
/// Call after construction when the model config specifies a
/// different resolution than the default 640x640.
2026-05-01 20:24:08 +10:00
///
/// Width/height are snapped UP to the nearest multiple of 32
/// (with a 64-pixel floor). Ultralytics YOLO necks (v8/v11/v12)
/// concat upsampled stride-32 features with backbone stride-16
/// features in /model.12/Concat — that op requires
/// 2*ceil(H/32) == ceil(H/16), which only holds when H is a
/// multiple of 32. Feeding e.g. H=80 fails at runtime with
/// "Axis 2 has mismatched dimensions of 5 and 6". The 64-pixel
/// floor avoids degenerate sub-stride-32 inputs that would
/// produce 1x1 P5 maps.
2026-03-28 16:54:11 +11:00
void setInputShape(int width, int height) {
2026-05-01 20:24:08 +10:00
constexpr int kStride = 32;
constexpr int kMinDim = 64;
auto snap = [](int v) {
if (v < kMinDim) v = kMinDim;
return ((v + kStride - 1) / kStride) * kStride;
};
inputImageShape = cv::Size(snap(width), snap(height));
2026-03-28 16:54:11 +11:00
}
/// True if the ONNX model has dynamic spatial dimensions.
bool hasDynamicInputShape() const { return isDynamicInputShape; }
private:
Ort::Value transform(const cv::Mat& mat) override;
Ort::Value transformBatch(const std::vector<cv::Mat>& images) override;
// ── Ultralytics-compatible letterbox ────────────────────────────
void letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool scaleUp = true,
int stride = 32);
// ── Detection postprocess ───────────────────────────────────────
std::vector<Object> postprocessEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── OBB postprocess ─────────────────────────────────────────────
std::vector<Object> postprocessOBBEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessOBBLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── Segmentation postprocess ────────────────────────────────────
std::vector<Object> postprocessSegEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessSegLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── Pose postprocess ────────────────────────────────────────────
std::vector<Object> postprocessPoseEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames,
float confThreshold, int numKPS);
std::vector<Object> postprocessPoseLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int numKPS, int maxDet = 300);
// ── Classification postprocess ──────────────────────────────────
std::vector<Object> postprocessClassify(
2026-04-08 13:45:52 +10:00
std::vector<Ort::Value>& outputTensors,
2026-03-28 16:54:11 +11:00
const std::vector<std::string>& classNames,
const cv::Size& imageSize);
// ── OBB NMS helpers (Prob-IoU based) ────────────────────────────
struct OrientedBox {
float x, y, width, height, angle;
};
static void getCovarianceComponents(const OrientedBox& box,
float& out1, float& out2, float& out3);
static std::vector<std::vector<float>> batchProbiou(
const std::vector<OrientedBox>& obb1,
const std::vector<OrientedBox>& obb2, float eps = 1e-7f);
static std::vector<int> nmsRotatedImpl(
const std::vector<OrientedBox>& sortedBoxes, float iouThreshold);
static std::vector<int> nmsRotated(
const std::vector<OrientedBox>& boxes,
const std::vector<float>& scores, float iouThreshold);
static std::vector<cv::Point2f> OBBToPoints(const OrientedBox& obb);
// ── Batch output slicing helper ────────────────────────────────
static Ort::Value sliceBatchOutput(
2026-04-08 13:45:52 +10:00
Ort::Value& batchTensor,
2026-03-28 16:54:11 +11:00
int64_t batchIndex,
const std::vector<int64_t>& fullShape,
Ort::MemoryInfo& memInfo);
// Cached model input shape
cv::Size inputImageShape;
bool isDynamicInputShape{ false };
2026-05-04 23:45:05 +10:00
// Per-call preprocess that writes into caller-provided buffer + dims
// instead of the BasicOrtHandler members (input_values_handler /
// input_node_dims). detect() / detectBatch() use these so multiple
// threads sharing one ONNXYOLO instance can preprocess and infer
// concurrently — the Ort input tensor's backing storage is unique
// per call and lives on the caller's stack. Without this, thread B
// could overwrite the shared buffer between thread A's transform()
// and thread A's Run(), causing A's Run() to consume B's pixels and
// return detections that belong to B's frame. Observed on
// Intel/OpenVINO EP where nothing else serializes.
Ort::Value transformInto(const cv::Mat& mat,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
Ort::Value transformBatchInto(const std::vector<cv::Mat>& images,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
2026-03-28 16:54:11 +11:00
};
// ====================================================================
// ANSONNXYOLO — ANSODBase wrapper for Ultralytics YOLO ONNX
//
// Compatible with all Ultralytics YOLO tasks and versions.
// Implements all required ANSODBase interfaces.
// ====================================================================
class ANSENGINE_API ANSONNXYOLO : public ANSODBase {
public:
2026-06-18 05:55:31 +10:00
// Explicit non-copy/non-move. The class has atomic + shared_ptr members
// that already implicitly delete these, but __declspec(dllexport)
// makes ClangCL eagerly instantiate the implicit copy/move ctor
// bodies; an explicit user-declared =delete short-circuits that.
ANSONNXYOLO(const ANSONNXYOLO&) = delete;
ANSONNXYOLO& operator=(const ANSONNXYOLO&) = delete;
ANSONNXYOLO(ANSONNXYOLO&&) = delete;
ANSONNXYOLO& operator=(ANSONNXYOLO&&) = delete;
ANSONNXYOLO() = default;
2026-03-28 16:54:11 +11:00
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath,
const std::string& modelZipPassword,
std::string& labelMap) override;
bool LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword) override;
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap) override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override;
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs,
const std::string& camera_id) override;
bool Destroy();
~ANSONNXYOLO();
private:
std::string _modelFilePath;
bool _modelLoadValid{ false };
// Filter thresholds
float PROBABILITY_THRESHOLD{ 0.25f };
float NMS_THRESHOLD{ 0.45f };
int TOP_K{ 300 };
// Pose estimation
int NUM_KPS{ 0 };
float KPS_THRESHOLD{ 0.5f };
2026-05-01 07:53:18 +10:00
// ONNX Runtime inference engine.
//
// B2: shared_ptr (was unique_ptr) so concurrent inference threads
// can grab a ref-counted local copy under the brief snapshot lock
// and dereference it OUTSIDE the lock without racing with a
// concurrent reset() in the DML-fatal CPU-fallback path. Without
// this, a thread doing `m_ortEngine->detect(...)` lock-free could
// see its underlying object freed mid-call by another thread that
// caught a DML E_FAIL and rebuilt the session — that was the
// 0xc0000005 access violation observed in ANSODEngine.dll under
// AMD/DirectML when multiple AI tasks ran in parallel.
2026-06-18 05:55:31 +10:00
// Elaborated `class ONNXYOLO` disambiguates from the
// ANSCENTER::ModelType::ONNXYOLO enumerator (ANSLicense.h) which
// shares the same unqualified name in this namespace.
std::shared_ptr<class ONNXYOLO> m_ortEngine;
2026-03-28 16:54:11 +11:00
2026-05-01 07:53:18 +10:00
// ────────────────────────────────────────────────────────────────
// DML stay-on-GPU recovery state.
//
// AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from
// DmlGraphFusionHelper / DmlCommandRecorder when the driver
// chokes under concurrent inference pressure. Empirically this
// is NOT a permanent device loss — recreating the ORT session on
// the same DML EP clears the wedged state and resumes GPU
// inference at full speed. Only the truly-fatal 887A0005
// (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback.
//
// Recovery contract (used by every Run() catch block):
// 1. compare_exchange on _dmlRecreateInProgress elects one
// thread as the recreate owner. Other threads return {}
// and retry on the next call.
// 2. The owner checks the cooldown — if we recreated in the
// last kDmlRecreateCooldownMs ms, skip (driver is in a
// prolonged bad state, don't thrash).
// 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit
// kMaxConsecutiveDmlFailures recreates without a successful
// inference in between, fall back to CPU as last resort.
// 4. Otherwise reset() + InitOrtEngine(<original DML EP>) +
// WarmUpEngine() to rebuild on the same GPU EP.
// 5. On successful inference, _dmlConsecutiveFailures resets
// to 0, so isolated transient hiccups don't accumulate
// toward the CPU-fallback threshold.
//
// This replaces the old single-shot `bool _dmlDeviceLost` flag
// which latched on the first failure and forced CPU forever —
// user requirement is to keep AI tasks running on GPU, not to
// trade GPU performance for stability the moment the driver
// hiccups once.
// ────────────────────────────────────────────────────────────────
std::atomic<bool> _dmlRecreateInProgress{ false };
std::atomic<int> _dmlConsecutiveFailures{ 0 };
std::chrono::steady_clock::time_point _lastDmlRecreateAt{};
// Tunables — tuned for AMD Radeon 780M observed behavior.
// threshold: catches isolated transient AMD DML hiccups. After
// 3 consecutive recreates without a successful inference, the
// driver is in a prolonged bad state and CPU is the only
// forward path.
// cooldown: prevents recreate thrash. AMD DML session creation
// takes 200–500 ms; if E_FAIL fires from many threads at once
// we don't want to recreate N times in a row.
static constexpr int kMaxConsecutiveDmlFailures = 3;
static constexpr int64_t kDmlRecreateCooldownMs = 5000;
// Centralised DML-fatal recovery used by all 3 catch sites.
// Caller must have already determined the exception is DML-related
// (IsDmlFatalError == true). msg is the exception message used
// both to log and to detect the device-removed sub-case. Returns
// immediately without doing anything when another thread already
// owns the recreate (CAS loser path).
void HandleDmlRunFailure(const char* context, const std::string& msg);
2026-04-09 08:09:02 +10:00
2026-03-28 16:54:11 +11:00
// Internal detection pipeline
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id);
// Internal batch detection pipeline
std::vector<std::vector<Object>> DetectObjectsBatch(
const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
// Initialise ORT engine from the resolved model path
bool InitOrtEngine();
2026-04-10 17:13:47 +10:00
// Run a few dummy inferences after session creation to force
// DirectML shader compile + kernel cache warm-up BEFORE the first
// real frame arrives. On AMD RDNA2 iGPUs (Radeon 680M) the first
// real inference can otherwise take several seconds of GPU time
// while compiling shaders, which has triggered amdkmdag.sys
// bugchecks under DirectML 1.15.4. Non-fatal on failure.
void WarmUpEngine();
2026-04-08 13:45:52 +10:00
public:
// Initialise ORT engine with explicit engine type override (e.g. CPU fallback for AMD iGPUs)
bool InitOrtEngine(ANSCENTER::EngineType engineType);
2026-03-28 16:54:11 +11:00
};
}
#endif