#ifndef ANSONNXYOLO_H #define ANSONNXYOLO_H #pragma once #include "ANSEngineCommon.h" #include "ONNXEngine.h" namespace ANSCENTER { // ==================================================================== // ONNXYOLO — Ultralytics YOLO inference via ONNX Runtime // // Compatible with ALL Ultralytics YOLO tasks and versions // (v8, v9, v10, v11, v26+) by auto-detecting the task type // from output tensor shapes at inference time. // // Supported tasks: // - Detection : [B,300,6] end2end or [B, nc+4, N] legacy // - OBB : [B,300,7] end2end or [B, nc+5, N] legacy // - Segmentation: [B,300,38]+protos or [B, nc+36, N]+protos // - Pose : [B,300,6+nk*3] or [B, nc+4+nk*3, N] legacy // - Classification: [B, nc] // // Preprocessing follows the exact Ultralytics LetterBox transform: // - center=True, stride=32, pad_value=114 // - Ultralytics-compatible -0.1/+0.1 rounding for deterministic padding // - BGR→RGB, /255.0 normalisation, HWC→CHW // ==================================================================== class ONNXENGINE_API ONNXYOLO : public BasicOrtHandler { public: explicit ONNXYOLO(const std::string& _onnx_path, unsigned int _num_threads = 1); explicit ONNXYOLO(const std::string& _onnx_path, EngineType engineType, unsigned int _num_threads = 1); ~ONNXYOLO() override = default; /// Run inference on a single image. /// Auto-detects the task type (detect/segment/obb/pose/classify) /// from the ONNX model's output tensor shapes. std::vector detect(const cv::Mat& image, const std::vector& classNames, float confThreshold = 0.25f, float iouThreshold = 0.45f, int numKPS = 0); /// True after detect() if the last inference was classification. bool lastWasClassification = false; /// Run batched inference on multiple images in a single ONNX session call. /// Falls back to sequential detect() if the model has fixed batch=1. std::vector> detectBatch( const std::vector& images, const std::vector& classNames, float confThreshold = 0.25f, float iouThreshold = 0.45f, int numKPS = 0); /// True after detectBatch() if the batch was classification. bool lastBatchWasClassification = false; /// Override the input image shape for dynamic-input models. /// Call after construction when the model config specifies a /// different resolution than the default 640x640. /// /// Width/height are snapped UP to the nearest multiple of 32 /// (with a 64-pixel floor). Ultralytics YOLO necks (v8/v11/v12) /// concat upsampled stride-32 features with backbone stride-16 /// features in /model.12/Concat — that op requires /// 2*ceil(H/32) == ceil(H/16), which only holds when H is a /// multiple of 32. Feeding e.g. H=80 fails at runtime with /// "Axis 2 has mismatched dimensions of 5 and 6". The 64-pixel /// floor avoids degenerate sub-stride-32 inputs that would /// produce 1x1 P5 maps. void setInputShape(int width, int height) { constexpr int kStride = 32; constexpr int kMinDim = 64; auto snap = [](int v) { if (v < kMinDim) v = kMinDim; return ((v + kStride - 1) / kStride) * kStride; }; inputImageShape = cv::Size(snap(width), snap(height)); } /// True if the ONNX model has dynamic spatial dimensions. bool hasDynamicInputShape() const { return isDynamicInputShape; } private: Ort::Value transform(const cv::Mat& mat) override; Ort::Value transformBatch(const std::vector& images) override; // ── Ultralytics-compatible letterbox ──────────────────────────── void letterBox(const cv::Mat& image, cv::Mat& outImage, const cv::Size& newShape, const cv::Scalar& color = cv::Scalar(114, 114, 114), bool scaleUp = true, int stride = 32); // ── Detection postprocess ─────────────────────────────────────── std::vector postprocessEndToEnd( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold); std::vector postprocessLegacy( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold, float iouThreshold, int maxDet = 300); // ── OBB postprocess ───────────────────────────────────────────── std::vector postprocessOBBEndToEnd( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold); std::vector postprocessOBBLegacy( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold, float iouThreshold, int maxDet = 300); // ── Segmentation postprocess ──────────────────────────────────── std::vector postprocessSegEndToEnd( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold); std::vector postprocessSegLegacy( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold, float iouThreshold, int maxDet = 300); // ── Pose postprocess ──────────────────────────────────────────── std::vector postprocessPoseEndToEnd( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold, int numKPS); std::vector postprocessPoseLegacy( const cv::Size& originalImageSize, const cv::Size& resizedImageShape, std::vector& outputTensors, const std::vector& classNames, float confThreshold, float iouThreshold, int numKPS, int maxDet = 300); // ── Classification postprocess ────────────────────────────────── std::vector postprocessClassify( std::vector& outputTensors, const std::vector& classNames, const cv::Size& imageSize); // ── OBB NMS helpers (Prob-IoU based) ──────────────────────────── struct OrientedBox { float x, y, width, height, angle; }; static void getCovarianceComponents(const OrientedBox& box, float& out1, float& out2, float& out3); static std::vector> batchProbiou( const std::vector& obb1, const std::vector& obb2, float eps = 1e-7f); static std::vector nmsRotatedImpl( const std::vector& sortedBoxes, float iouThreshold); static std::vector nmsRotated( const std::vector& boxes, const std::vector& scores, float iouThreshold); static std::vector OBBToPoints(const OrientedBox& obb); // ── Batch output slicing helper ──────────────────────────────── static Ort::Value sliceBatchOutput( Ort::Value& batchTensor, int64_t batchIndex, const std::vector& fullShape, Ort::MemoryInfo& memInfo); // Cached model input shape cv::Size inputImageShape; bool isDynamicInputShape{ false }; // Per-call preprocess that writes into caller-provided buffer + dims // instead of the BasicOrtHandler members (input_values_handler / // input_node_dims). detect() / detectBatch() use these so multiple // threads sharing one ONNXYOLO instance can preprocess and infer // concurrently — the Ort input tensor's backing storage is unique // per call and lives on the caller's stack. Without this, thread B // could overwrite the shared buffer between thread A's transform() // and thread A's Run(), causing A's Run() to consume B's pixels and // return detections that belong to B's frame. Observed on // Intel/OpenVINO EP where nothing else serializes. Ort::Value transformInto(const cv::Mat& mat, std::vector& outBuffer, std::vector& outDims); Ort::Value transformBatchInto(const std::vector& images, std::vector& outBuffer, std::vector& outDims); }; // ==================================================================== // ANSONNXYOLO — ANSODBase wrapper for Ultralytics YOLO ONNX // // Compatible with all Ultralytics YOLO tasks and versions. // Implements all required ANSODBase interfaces. // ==================================================================== class ANSENGINE_API ANSONNXYOLO : public ANSODBase { public: // Explicit non-copy/non-move. The class has atomic + shared_ptr members // that already implicitly delete these, but __declspec(dllexport) // makes ClangCL eagerly instantiate the implicit copy/move ctor // bodies; an explicit user-declared =delete short-circuits that. ANSONNXYOLO(const ANSONNXYOLO&) = delete; ANSONNXYOLO& operator=(const ANSONNXYOLO&) = delete; ANSONNXYOLO(ANSONNXYOLO&&) = delete; ANSONNXYOLO& operator=(ANSONNXYOLO&&) = delete; ANSONNXYOLO() = default; bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override; bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override; bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) override; bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override; std::vector RunInference(const cv::Mat& input); std::vector RunInference(const cv::Mat& input, const std::string& camera_id); std::vector> RunInferencesBatch( const std::vector& inputs, const std::string& camera_id) override; bool Destroy(); ~ANSONNXYOLO(); private: std::string _modelFilePath; bool _modelLoadValid{ false }; // Filter thresholds float PROBABILITY_THRESHOLD{ 0.25f }; float NMS_THRESHOLD{ 0.45f }; int TOP_K{ 300 }; // Pose estimation int NUM_KPS{ 0 }; float KPS_THRESHOLD{ 0.5f }; // ONNX Runtime inference engine. // // B2: shared_ptr (was unique_ptr) so concurrent inference threads // can grab a ref-counted local copy under the brief snapshot lock // and dereference it OUTSIDE the lock without racing with a // concurrent reset() in the DML-fatal CPU-fallback path. Without // this, a thread doing `m_ortEngine->detect(...)` lock-free could // see its underlying object freed mid-call by another thread that // caught a DML E_FAIL and rebuilt the session — that was the // 0xc0000005 access violation observed in ANSODEngine.dll under // AMD/DirectML when multiple AI tasks ran in parallel. // Elaborated `class ONNXYOLO` disambiguates from the // ANSCENTER::ModelType::ONNXYOLO enumerator (ANSLicense.h) which // shares the same unqualified name in this namespace. std::shared_ptr m_ortEngine; // ──────────────────────────────────────────────────────────────── // DML stay-on-GPU recovery state. // // AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from // DmlGraphFusionHelper / DmlCommandRecorder when the driver // chokes under concurrent inference pressure. Empirically this // is NOT a permanent device loss — recreating the ORT session on // the same DML EP clears the wedged state and resumes GPU // inference at full speed. Only the truly-fatal 887A0005 // (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback. // // Recovery contract (used by every Run() catch block): // 1. compare_exchange on _dmlRecreateInProgress elects one // thread as the recreate owner. Other threads return {} // and retry on the next call. // 2. The owner checks the cooldown — if we recreated in the // last kDmlRecreateCooldownMs ms, skip (driver is in a // prolonged bad state, don't thrash). // 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit // kMaxConsecutiveDmlFailures recreates without a successful // inference in between, fall back to CPU as last resort. // 4. Otherwise reset() + InitOrtEngine() + // WarmUpEngine() to rebuild on the same GPU EP. // 5. On successful inference, _dmlConsecutiveFailures resets // to 0, so isolated transient hiccups don't accumulate // toward the CPU-fallback threshold. // // This replaces the old single-shot `bool _dmlDeviceLost` flag // which latched on the first failure and forced CPU forever — // user requirement is to keep AI tasks running on GPU, not to // trade GPU performance for stability the moment the driver // hiccups once. // ──────────────────────────────────────────────────────────────── std::atomic _dmlRecreateInProgress{ false }; std::atomic _dmlConsecutiveFailures{ 0 }; std::chrono::steady_clock::time_point _lastDmlRecreateAt{}; // Tunables — tuned for AMD Radeon 780M observed behavior. // threshold: catches isolated transient AMD DML hiccups. After // 3 consecutive recreates without a successful inference, the // driver is in a prolonged bad state and CPU is the only // forward path. // cooldown: prevents recreate thrash. AMD DML session creation // takes 200–500 ms; if E_FAIL fires from many threads at once // we don't want to recreate N times in a row. static constexpr int kMaxConsecutiveDmlFailures = 3; static constexpr int64_t kDmlRecreateCooldownMs = 5000; // Centralised DML-fatal recovery used by all 3 catch sites. // Caller must have already determined the exception is DML-related // (IsDmlFatalError == true). msg is the exception message used // both to log and to detect the device-removed sub-case. Returns // immediately without doing anything when another thread already // owns the recreate (CAS loser path). void HandleDmlRunFailure(const char* context, const std::string& msg); // Internal detection pipeline std::vector DetectObjects(const cv::Mat& inputImage, const std::string& camera_id); // Internal batch detection pipeline std::vector> DetectObjectsBatch( const std::vector& inputImages, const std::string& camera_id); // Initialise ORT engine from the resolved model path bool InitOrtEngine(); // Run a few dummy inferences after session creation to force // DirectML shader compile + kernel cache warm-up BEFORE the first // real frame arrives. On AMD RDNA2 iGPUs (Radeon 680M) the first // real inference can otherwise take several seconds of GPU time // while compiling shaders, which has triggered amdkmdag.sys // bugchecks under DirectML 1.15.4. Non-fatal on failure. void WarmUpEngine(); public: // Initialise ORT engine with explicit engine type override (e.g. CPU fallback for AMD iGPUs) bool InitOrtEngine(ANSCENTER::EngineType engineType); }; } #endif