Files
ANSCORE/modules/ANSODEngine/ANSONNXYOLO.h
T

340 lines
17 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#ifndef ANSONNXYOLO_H
#define ANSONNXYOLO_H
#pragma once
#include "ANSEngineCommon.h"
#include "ONNXEngine.h"
namespace ANSCENTER {
// ====================================================================
// ONNXYOLO — Ultralytics YOLO inference via ONNX Runtime
//
// Compatible with ALL Ultralytics YOLO tasks and versions
// (v8, v9, v10, v11, v26+) by auto-detecting the task type
// from output tensor shapes at inference time.
//
// Supported tasks:
// - Detection : [B,300,6] end2end or [B, nc+4, N] legacy
// - OBB : [B,300,7] end2end or [B, nc+5, N] legacy
// - Segmentation: [B,300,38]+protos or [B, nc+36, N]+protos
// - Pose : [B,300,6+nk*3] or [B, nc+4+nk*3, N] legacy
// - Classification: [B, nc]
//
// Preprocessing follows the exact Ultralytics LetterBox transform:
// - center=True, stride=32, pad_value=114
// - Ultralytics-compatible -0.1/+0.1 rounding for deterministic padding
// - BGR→RGB, /255.0 normalisation, HWC→CHW
// ====================================================================
class ONNXENGINE_API ONNXYOLO : public BasicOrtHandler
{
public:
explicit ONNXYOLO(const std::string& _onnx_path,
unsigned int _num_threads = 1);
explicit ONNXYOLO(const std::string& _onnx_path,
EngineType engineType,
unsigned int _num_threads = 1);
~ONNXYOLO() override = default;
/// Run inference on a single image.
/// Auto-detects the task type (detect/segment/obb/pose/classify)
/// from the ONNX model's output tensor shapes.
std::vector<Object> detect(const cv::Mat& image,
const std::vector<std::string>& classNames,
float confThreshold = 0.25f,
float iouThreshold = 0.45f,
int numKPS = 0);
/// True after detect() if the last inference was classification.
bool lastWasClassification = false;
/// Run batched inference on multiple images in a single ONNX session call.
/// Falls back to sequential detect() if the model has fixed batch=1.
std::vector<std::vector<Object>> detectBatch(
const std::vector<cv::Mat>& images,
const std::vector<std::string>& classNames,
float confThreshold = 0.25f,
float iouThreshold = 0.45f,
int numKPS = 0);
/// True after detectBatch() if the batch was classification.
bool lastBatchWasClassification = false;
/// Override the input image shape for dynamic-input models.
/// Call after construction when the model config specifies a
/// different resolution than the default 640x640.
///
/// Width/height are snapped UP to the nearest multiple of 32
/// (with a 64-pixel floor). Ultralytics YOLO necks (v8/v11/v12)
/// concat upsampled stride-32 features with backbone stride-16
/// features in /model.12/Concat — that op requires
/// 2*ceil(H/32) == ceil(H/16), which only holds when H is a
/// multiple of 32. Feeding e.g. H=80 fails at runtime with
/// "Axis 2 has mismatched dimensions of 5 and 6". The 64-pixel
/// floor avoids degenerate sub-stride-32 inputs that would
/// produce 1x1 P5 maps.
void setInputShape(int width, int height) {
constexpr int kStride = 32;
constexpr int kMinDim = 64;
auto snap = [](int v) {
if (v < kMinDim) v = kMinDim;
return ((v + kStride - 1) / kStride) * kStride;
};
inputImageShape = cv::Size(snap(width), snap(height));
}
/// True if the ONNX model has dynamic spatial dimensions.
bool hasDynamicInputShape() const { return isDynamicInputShape; }
private:
Ort::Value transform(const cv::Mat& mat) override;
Ort::Value transformBatch(const std::vector<cv::Mat>& images) override;
// ── Ultralytics-compatible letterbox ────────────────────────────
void letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool scaleUp = true,
int stride = 32);
// ── Detection postprocess ───────────────────────────────────────
std::vector<Object> postprocessEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── OBB postprocess ─────────────────────────────────────────────
std::vector<Object> postprocessOBBEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessOBBLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── Segmentation postprocess ────────────────────────────────────
std::vector<Object> postprocessSegEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessSegLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── Pose postprocess ────────────────────────────────────────────
std::vector<Object> postprocessPoseEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, int numKPS);
std::vector<Object> postprocessPoseLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int numKPS, int maxDet = 300);
// ── Classification postprocess ──────────────────────────────────
std::vector<Object> postprocessClassify(
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
const cv::Size& imageSize);
// ── OBB NMS helpers (Prob-IoU based) ────────────────────────────
struct OrientedBox {
float x, y, width, height, angle;
};
static void getCovarianceComponents(const OrientedBox& box,
float& out1, float& out2, float& out3);
static std::vector<std::vector<float>> batchProbiou(
const std::vector<OrientedBox>& obb1,
const std::vector<OrientedBox>& obb2, float eps = 1e-7f);
static std::vector<int> nmsRotatedImpl(
const std::vector<OrientedBox>& sortedBoxes, float iouThreshold);
static std::vector<int> nmsRotated(
const std::vector<OrientedBox>& boxes,
const std::vector<float>& scores, float iouThreshold);
static std::vector<cv::Point2f> OBBToPoints(const OrientedBox& obb);
// ── Batch output slicing helper ────────────────────────────────
static Ort::Value sliceBatchOutput(
Ort::Value& batchTensor,
int64_t batchIndex,
const std::vector<int64_t>& fullShape,
Ort::MemoryInfo& memInfo);
// Cached model input shape
cv::Size inputImageShape;
bool isDynamicInputShape{ false };
// Per-call preprocess that writes into caller-provided buffer + dims
// instead of the BasicOrtHandler members (input_values_handler /
// input_node_dims). detect() / detectBatch() use these so multiple
// threads sharing one ONNXYOLO instance can preprocess and infer
// concurrently — the Ort input tensor's backing storage is unique
// per call and lives on the caller's stack. Without this, thread B
// could overwrite the shared buffer between thread A's transform()
// and thread A's Run(), causing A's Run() to consume B's pixels and
// return detections that belong to B's frame. Observed on
// Intel/OpenVINO EP where nothing else serializes.
Ort::Value transformInto(const cv::Mat& mat,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
Ort::Value transformBatchInto(const std::vector<cv::Mat>& images,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
};
// ====================================================================
// ANSONNXYOLO — ANSODBase wrapper for Ultralytics YOLO ONNX
//
// Compatible with all Ultralytics YOLO tasks and versions.
// Implements all required ANSODBase interfaces.
// ====================================================================
class ANSENGINE_API ANSONNXYOLO : public ANSODBase {
public:
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath,
const std::string& modelZipPassword,
std::string& labelMap) override;
bool LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword) override;
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap) override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override;
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs,
const std::string& camera_id) override;
bool Destroy();
~ANSONNXYOLO();
private:
std::string _modelFilePath;
bool _modelLoadValid{ false };
// Filter thresholds
float PROBABILITY_THRESHOLD{ 0.25f };
float NMS_THRESHOLD{ 0.45f };
int TOP_K{ 300 };
// Pose estimation
int NUM_KPS{ 0 };
float KPS_THRESHOLD{ 0.5f };
// ONNX Runtime inference engine.
//
// B2: shared_ptr (was unique_ptr) so concurrent inference threads
// can grab a ref-counted local copy under the brief snapshot lock
// and dereference it OUTSIDE the lock without racing with a
// concurrent reset() in the DML-fatal CPU-fallback path. Without
// this, a thread doing `m_ortEngine->detect(...)` lock-free could
// see its underlying object freed mid-call by another thread that
// caught a DML E_FAIL and rebuilt the session — that was the
// 0xc0000005 access violation observed in ANSODEngine.dll under
// AMD/DirectML when multiple AI tasks ran in parallel.
std::shared_ptr<ONNXYOLO> m_ortEngine;
// ────────────────────────────────────────────────────────────────
// DML stay-on-GPU recovery state.
//
// AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from
// DmlGraphFusionHelper / DmlCommandRecorder when the driver
// chokes under concurrent inference pressure. Empirically this
// is NOT a permanent device loss — recreating the ORT session on
// the same DML EP clears the wedged state and resumes GPU
// inference at full speed. Only the truly-fatal 887A0005
// (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback.
//
// Recovery contract (used by every Run() catch block):
// 1. compare_exchange on _dmlRecreateInProgress elects one
// thread as the recreate owner. Other threads return {}
// and retry on the next call.
// 2. The owner checks the cooldown — if we recreated in the
// last kDmlRecreateCooldownMs ms, skip (driver is in a
// prolonged bad state, don't thrash).
// 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit
// kMaxConsecutiveDmlFailures recreates without a successful
// inference in between, fall back to CPU as last resort.
// 4. Otherwise reset() + InitOrtEngine(<original DML EP>) +
// WarmUpEngine() to rebuild on the same GPU EP.
// 5. On successful inference, _dmlConsecutiveFailures resets
// to 0, so isolated transient hiccups don't accumulate
// toward the CPU-fallback threshold.
//
// This replaces the old single-shot `bool _dmlDeviceLost` flag
// which latched on the first failure and forced CPU forever —
// user requirement is to keep AI tasks running on GPU, not to
// trade GPU performance for stability the moment the driver
// hiccups once.
// ────────────────────────────────────────────────────────────────
std::atomic<bool> _dmlRecreateInProgress{ false };
std::atomic<int> _dmlConsecutiveFailures{ 0 };
std::chrono::steady_clock::time_point _lastDmlRecreateAt{};
// Tunables — tuned for AMD Radeon 780M observed behavior.
// threshold: catches isolated transient AMD DML hiccups. After
// 3 consecutive recreates without a successful inference, the
// driver is in a prolonged bad state and CPU is the only
// forward path.
// cooldown: prevents recreate thrash. AMD DML session creation
// takes 200–500 ms; if E_FAIL fires from many threads at once
// we don't want to recreate N times in a row.
static constexpr int kMaxConsecutiveDmlFailures = 3;
static constexpr int64_t kDmlRecreateCooldownMs = 5000;
// Centralised DML-fatal recovery used by all 3 catch sites.
// Caller must have already determined the exception is DML-related
// (IsDmlFatalError == true). msg is the exception message used
// both to log and to detect the device-removed sub-case. Returns
// immediately without doing anything when another thread already
// owns the recreate (CAS loser path).
void HandleDmlRunFailure(const char* context, const std::string& msg);
// Internal detection pipeline
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id);
// Internal batch detection pipeline
std::vector<std::vector<Object>> DetectObjectsBatch(
const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
// Initialise ORT engine from the resolved model path
bool InitOrtEngine();
// Run a few dummy inferences after session creation to force
// DirectML shader compile + kernel cache warm-up BEFORE the first
// real frame arrives. On AMD RDNA2 iGPUs (Radeon 680M) the first
// real inference can otherwise take several seconds of GPU time
// while compiling shaders, which has triggered amdkmdag.sys
// bugchecks under DirectML 1.15.4. Non-fatal on failure.
void WarmUpEngine();
public:
// Initialise ORT engine with explicit engine type override (e.g. CPU fallback for AMD iGPUs)
bool InitOrtEngine(ANSCENTER::EngineType engineType);
};
}
#endif