Fix AMD ALPR_OCR and custom model
This commit is contained in:
@@ -24,7 +24,7 @@ namespace ANSCENTER {
|
||||
std::unique_ptr<IANSCustomClass> _customDetector;
|
||||
HMODULE hMod= nullptr;
|
||||
std::string FindANSCustomLibraryName(const std::string& folderPath, const std::string& keyword);
|
||||
bool CreateCustomDetector();
|
||||
bool CreateCustomDetector();
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@@ -36,6 +36,57 @@ const int MAX_MISSING_SCREEN = 1000;
|
||||
|
||||
namespace ANSCENTER
|
||||
{
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// DirectML fatal-error detection
|
||||
//
|
||||
// AMD's DirectML EP can fail in several distinct ways that all leave the
|
||||
// Ort::Session permanently broken and require a fresh CPU-fallback session:
|
||||
//
|
||||
// 887A0005 DXGI_ERROR_DEVICE_REMOVED — adapter killed (RDP attach,
|
||||
// TDR, sleep/wake, driver swap). GetDeviceRemovedReason gives
|
||||
// the sub-cause but the device is gone either way.
|
||||
// 80004005 E_FAIL — generic D3D12 / DML failure. Seen from
|
||||
// DmlGraphFusionHelper::ExecuteOperator and
|
||||
// DmlCommandRecorder::ExecuteCommandList on AMD when the
|
||||
// driver couldn't compile/execute a fused subgraph or when
|
||||
// TDR fires mid-command-list. No structured sub-cause
|
||||
// available — must treat as device-fatal.
|
||||
//
|
||||
// Branding strings ("DmlFusedNode", "DmlGraphFusionHelper",
|
||||
// "DmlCommandRecorder") catch future failure modes that surface the same
|
||||
// way without us having to chase each new HRESULT.
|
||||
//
|
||||
// Used by every catch site that wraps Ort::Session::Run on AMD DML.
|
||||
// Centralised here so a new failure mode is handled once, not 4+ times.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
inline bool IsDmlFatalError(const std::string& msg) {
|
||||
return msg.find("887A0005") != std::string::npos
|
||||
|| msg.find("80004005") != std::string::npos
|
||||
|| msg.find("DmlFusedNode") != std::string::npos
|
||||
|| msg.find("DmlGraphFusionHelper") != std::string::npos
|
||||
|| msg.find("DmlCommandRecorder") != std::string::npos
|
||||
|| msg.find("DmlExecutionProvider") != std::string::npos;
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// IsDmlDeviceRemoved — narrowly checks ONLY 887A0005
|
||||
// (DXGI_ERROR_DEVICE_REMOVED). This is the truly unrecoverable case:
|
||||
// the D3D12 adapter is gone (driver crashed, RDP attach killed the
|
||||
// device, sleep/wake torched the WDDM context). No amount of session
|
||||
// recreation will bring it back — only a process restart or CPU
|
||||
// fallback can recover.
|
||||
//
|
||||
// In contrast, 80004005 (E_FAIL) from DmlGraphFusionHelper /
|
||||
// DmlCommandRecorder is intermittent on AMD when the driver chokes on
|
||||
// a fused subgraph or TDR fires mid-command-list. A fresh DML
|
||||
// session typically clears it and resumes GPU inference — that's the
|
||||
// stay-on-GPU recovery path used by ANSONNXYOLO's catch blocks.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
inline bool IsDmlDeviceRemoved(const std::string& msg) {
|
||||
return msg.find("887A0005") != std::string::npos
|
||||
|| msg.find("DXGI_ERROR_DEVICE_REMOVED") != std::string::npos;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
typename std::enable_if<std::is_arithmetic<T>::value, T>::type
|
||||
inline clamp(const T& value, const T& low, const T& high)
|
||||
|
||||
+113
-197
@@ -1,4 +1,5 @@
|
||||
#include"ANSONNXCL.h"
|
||||
#include "ANSONNXYOLO.h" // ANSONNXYOLO is the delegate target — see facade comment in ANSONNXCL.h
|
||||
#include "EPLoader.h"
|
||||
#include "OpenVINODeviceConfig.h"
|
||||
#include <typeinfo> // typeid(e).name() in ANS_DBG diagnostics
|
||||
@@ -876,9 +877,50 @@ namespace ANSCENTER
|
||||
);
|
||||
}
|
||||
catch (const Ort::Exception& e) {
|
||||
const std::string msg = e.what();
|
||||
this->_logger.LogError("ANSONNXCL::classify",
|
||||
"ONNX Runtime Exception during Run(): " + std::string(e.what()), __FILE__, __LINE__);
|
||||
"ONNX Runtime Exception during Run(): " + msg, __FILE__, __LINE__);
|
||||
delete[] blobPtr; // Clean up on error
|
||||
blobPtr = nullptr;
|
||||
|
||||
// DML device-lost / DML graph-execution failure → tear down the
|
||||
// dead DirectML session and rebuild on CPU. Without this, every
|
||||
// subsequent classify() call hits the same dead session and
|
||||
// floods the event log (16k+ entries observed on AMD/RDP).
|
||||
if (IsDmlFatalError(msg) && !_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXCL::classify",
|
||||
std::string("DirectML fatal error — recreating session on CPU. msg=")
|
||||
+ msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSONNXCL", "DML fatal — rebuild on CPU msg=%s", msg.c_str());
|
||||
try {
|
||||
// Order matters: drop AllocatedStringPtrs (which depend
|
||||
// on the live session's allocator) BEFORE tearing down
|
||||
// session_, sessionOptions_, env_. Init() will repopulate.
|
||||
inputNodeNameAllocatedStrings_.clear();
|
||||
outputNodeNameAllocatedStrings_.clear();
|
||||
inputNames_.clear();
|
||||
outputNames_.clear();
|
||||
session_ = Ort::Session{ nullptr };
|
||||
sessionOptions_ = Ort::SessionOptions{ nullptr };
|
||||
env_ = Ort::Env{ nullptr };
|
||||
if (Init(_modelFilePath, inputImageShape_, /*useGPU=*/false)) {
|
||||
_logger.LogInfo("ANSONNXCL::classify",
|
||||
"CPU fallback session created successfully",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSONNXCL", "CPU fallback OK");
|
||||
} else {
|
||||
_logger.LogFatal("ANSONNXCL::classify",
|
||||
"CPU fallback session creation failed",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
} catch (const std::exception& re) {
|
||||
_logger.LogFatal("ANSONNXCL::classify",
|
||||
std::string("CPU fallback exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -922,216 +964,90 @@ namespace ANSCENTER
|
||||
return {};
|
||||
}
|
||||
}
|
||||
bool ANSONNXCL::OptimizeModel(bool fp16, std::string & optimizedModelFolder) {
|
||||
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
_modelLoadValid = false;
|
||||
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
|
||||
if (!result) return false;
|
||||
// Parsing for YOLO only here
|
||||
_modelConfig = modelConfig;
|
||||
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
|
||||
_modelConfig.modelType = ModelType::ONNXCL;
|
||||
_modelConfig.inpHeight = 224;
|
||||
_modelConfig.inpWidth = 224;
|
||||
if (_modelConfig.modelMNSThreshold < 0.2)
|
||||
_modelConfig.modelMNSThreshold = 0.5;
|
||||
if (_modelConfig.modelConfThreshold < 0.2)
|
||||
_modelConfig.modelConfThreshold = 0.5;
|
||||
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
|
||||
_modelConfig.numKPS = 17;
|
||||
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
|
||||
_fp16 = (modelConfig.precisionType == PrecisionType::FP16);
|
||||
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// Public-method facade: delegate everything to the wrapped ANSONNXYOLO.
|
||||
//
|
||||
// This is a verification harness — ANSONNXYOLO already covers the
|
||||
// classification path (auto-detected from model output shape) and ships
|
||||
// with the safe DML config + DML-fatal-error recovery. By routing every
|
||||
// ANSONNXCL public call through ANSONNXYOLO we can A/B test whether
|
||||
// ANSONNXCL's own session/postprocess logic is the actual cause of the
|
||||
// AMD same-process DML races we saw with the custom helmet model.
|
||||
//
|
||||
// The dead code below the facade (Init / preprocess / postprocess /
|
||||
// classify / Ort::* members in the header) is intentionally preserved
|
||||
// so the original implementation is one git-revert away if needed.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
namespace {
|
||||
// Lazy-create the delegate on first public call. Not thread-safe in
|
||||
// the strict sense, but the public methods all take _mutex first via
|
||||
// the underlying ANSODBase, so concurrent first-time creation is
|
||||
// serialised in practice.
|
||||
inline ANSONNXYOLO& GetDelegate(std::unique_ptr<ANSONNXYOLO>& slot) {
|
||||
if (!slot) {
|
||||
slot = std::make_unique<ANSONNXYOLO>();
|
||||
ANS_DBG("ANSONNXCL_Facade",
|
||||
"delegate created (ANSONNXYOLO=%p)", (void*)slot.get());
|
||||
}
|
||||
else {// This is old version of model zip file
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValidFileName(_classFilePath);
|
||||
if (!isValidFileName)
|
||||
{
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromString();
|
||||
}
|
||||
else {
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
// 1. Load labelMap and engine
|
||||
labelMap.clear();
|
||||
if (!_classes.empty())
|
||||
labelMap = VectorToCommaSeparatedString(_classes);
|
||||
|
||||
// 2. Initialize ONNX Runtime session
|
||||
cv::Size targetInputShape= cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
|
||||
result = Init(_modelFilePath, targetInputShape,true);
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::Initialize", e.what(), __FILE__, __LINE__);
|
||||
return false;
|
||||
return *slot;
|
||||
}
|
||||
}
|
||||
bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
|
||||
if (!result) return false;
|
||||
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
|
||||
_modelConfig.modelType = ModelType::TENSORRT;
|
||||
_modelConfig.inpHeight = 224;
|
||||
_modelConfig.inpWidth = 224;
|
||||
if (_modelConfig.modelMNSThreshold < 0.2)
|
||||
_modelConfig.modelMNSThreshold = 0.5;
|
||||
if (_modelConfig.modelConfThreshold < 0.2)
|
||||
_modelConfig.modelConfThreshold = 0.5;
|
||||
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
|
||||
_modelConfig.numKPS = 17;
|
||||
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
|
||||
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
|
||||
_fp16 = true; // Load Model from Here
|
||||
|
||||
// 0. Check if the configuration file exist
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
}
|
||||
else {// This is old version of model zip file
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValidFileName(_classFilePath);
|
||||
if (!isValidFileName)
|
||||
{
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromString();
|
||||
}
|
||||
else {
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
// Initialize ONNX Runtime session
|
||||
cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
|
||||
result = Init(_modelFilePath, targetInputShape,true);
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::LoadModel", e.what(), __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
bool ANSONNXCL::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "OptimizeModel → ANSONNXYOLO");
|
||||
return GetDelegate(_yoloDelegate).OptimizeModel(fp16, optimizedModelFolder);
|
||||
}
|
||||
bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
ModelLoadingGuard mlg(_modelLoading);
|
||||
try {
|
||||
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
|
||||
if (!result) return false;
|
||||
std::string _modelName = modelName;
|
||||
if (_modelName.empty()) {
|
||||
_modelName = "train_last";
|
||||
}
|
||||
std::string modelFullName = _modelName + ".onnx";
|
||||
// Parsing for YOLO only here
|
||||
_modelConfig = modelConfig;
|
||||
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
|
||||
_modelConfig.modelType = ModelType::TENSORRT;
|
||||
_modelConfig.inpHeight = 224;
|
||||
_modelConfig.inpWidth = 224;
|
||||
if (_modelConfig.modelMNSThreshold < 0.2)
|
||||
_modelConfig.modelMNSThreshold = 0.5;
|
||||
if (_modelConfig.modelConfThreshold < 0.2)
|
||||
_modelConfig.modelConfThreshold = 0.5;
|
||||
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
|
||||
_modelConfig.numKPS = 17;
|
||||
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
|
||||
_fp16 = true; // Load Model from Here
|
||||
|
||||
// 0. Check if the configuration file exist
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
}
|
||||
else {// This is old version of model zip file
|
||||
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
|
||||
_classFilePath = CreateFilePath(_modelFolder, className);
|
||||
std::ifstream isValidFileName(_classFilePath);
|
||||
if (!isValidFileName)
|
||||
{
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromString();
|
||||
}
|
||||
else {
|
||||
this->_logger.LogDebug("ANSONNXCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
|
||||
LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
// 1. Load labelMap and engine
|
||||
labelMap.clear();
|
||||
if (!_classes.empty())
|
||||
labelMap = VectorToCommaSeparatedString(_classes);
|
||||
// 2. Initialize ONNX Runtime session
|
||||
cv::Size targetInputShape = cv::Size(_modelConfig.inpWidth, _modelConfig.inpHeight);
|
||||
result = Init(_modelFilePath, targetInputShape,true);
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
bool ANSONNXCL::Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||
const std::string& modelZipFilePath,
|
||||
const std::string& modelZipPassword,
|
||||
std::string& labelMap) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "Initialize → ANSONNXYOLO zip=%s",
|
||||
modelZipFilePath.c_str());
|
||||
return GetDelegate(_yoloDelegate).Initialize(
|
||||
licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
|
||||
}
|
||||
|
||||
bool ANSONNXCL::LoadModel(const std::string& modelZipFilePath,
|
||||
const std::string& modelZipPassword) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "LoadModel → ANSONNXYOLO zip=%s",
|
||||
modelZipFilePath.c_str());
|
||||
return GetDelegate(_yoloDelegate).LoadModel(modelZipFilePath, modelZipPassword);
|
||||
}
|
||||
|
||||
bool ANSONNXCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
|
||||
std::string modelName, std::string className,
|
||||
const std::string& modelFolder,
|
||||
std::string& labelMap) {
|
||||
ANS_DBG("ANSONNXCL_Facade",
|
||||
"LoadModelFromFolder → ANSONNXYOLO modelName=%s folder=%s",
|
||||
modelName.c_str(), modelFolder.c_str());
|
||||
return GetDelegate(_yoloDelegate).LoadModelFromFolder(
|
||||
licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
|
||||
}
|
||||
|
||||
bool ANSONNXCL::Destroy() {
|
||||
return true;
|
||||
if (_yoloDelegate) {
|
||||
ANS_DBG("ANSONNXCL_Facade", "Destroy → ANSONNXYOLO");
|
||||
return _yoloDelegate->Destroy();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
ANSONNXCL::~ANSONNXCL() {
|
||||
// Explicit Destroy() so the delegate's session is torn down before
|
||||
// the unique_ptr destructor frees the object. unique_ptr<ANSONNXYOLO>
|
||||
// still releases the heap memory afterwards.
|
||||
Destroy();
|
||||
}
|
||||
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& input, const std::string& camera_id) {
|
||||
if (!PreInferenceCheck("ANSONNXCL::RunInference")) return {};
|
||||
try {
|
||||
std::vector<Object> result;
|
||||
if (input.empty()) return result;
|
||||
if ((input.cols < 5) || (input.rows < 5)) return result;
|
||||
return classify(input, camera_id);
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
this->_logger.LogFatal("ANSONNXCL::RunInference", e.what(), __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& input,
|
||||
const std::string& camera_id) {
|
||||
if (input.empty()) return {};
|
||||
if (input.cols < 5 || input.rows < 5) return {};
|
||||
return GetDelegate(_yoloDelegate).RunInference(input, camera_id);
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXCL::RunInference(const cv::Mat& inputImgBGR) {
|
||||
return RunInference(inputImgBGR, "CustomCam");
|
||||
}
|
||||
|
||||
@@ -3,8 +3,26 @@
|
||||
#pragma once
|
||||
#include "ANSEngineCommon.h"
|
||||
#include "engine.h"
|
||||
#include <memory>
|
||||
|
||||
namespace ANSCENTER {
|
||||
// Forward declaration: avoid pulling ANSONNXYOLO.h into every consumer of
|
||||
// ANSONNXCL.h. Definition is included in ANSONNXCL.cpp.
|
||||
class ANSONNXYOLO;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
// ANSONNXCL — verification facade over ANSONNXYOLO.
|
||||
//
|
||||
// ANSONNXYOLO already auto-detects classification models (single 2D
|
||||
// output [B, nc]) and routes through its own classification postprocess.
|
||||
// It also has the safe DML config (DisableMemPattern + ORT_SEQUENTIAL +
|
||||
// EXTENDED) and the IsDmlFatalError catch-and-rebuild path.
|
||||
//
|
||||
// We delegate every public method here to an internal ANSONNXYOLO so we
|
||||
// can A/B test whether ANSONNXCL's own session/postprocess code is the
|
||||
// real cause of the AMD same-process DML races we saw with the custom
|
||||
// helmet model. Public ABI is unchanged — external callers don't move.
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
class ANSENGINE_API ANSONNXCL :public ANSODBase {
|
||||
public:
|
||||
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
|
||||
@@ -15,10 +33,23 @@ namespace ANSCENTER {
|
||||
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
|
||||
bool Destroy();
|
||||
~ANSONNXCL();
|
||||
private:
|
||||
// The wrapped engine. Lazy-initialised on first public call so we
|
||||
// don't pay the cost (and pull in the dependency) for callers who
|
||||
// never use this class.
|
||||
std::unique_ptr<ANSONNXYOLO> _yoloDelegate;
|
||||
private:
|
||||
std::string _modelFilePath;
|
||||
bool _modelLoadValid;
|
||||
bool _fp16{ false };
|
||||
|
||||
// DML device-lost recovery: when DirectML fails fatally
|
||||
// (HRESULT 887A0005 device-removed, 80004005 graph/command-recorder
|
||||
// E_FAIL, etc. — see ANSCENTER::IsDmlFatalError) the Ort::Session
|
||||
// is permanently broken. We detect once, recreate on CPU via
|
||||
// Init(_modelFilePath, inputImageShape_, /*useGPU=*/false), and
|
||||
// suppress further error-log flooding via this flag.
|
||||
bool _dmlDeviceLost{ false };
|
||||
size_t vectorProduct(const std::vector<int64_t>& vector);
|
||||
bool preprocessImageToTensor(const cv::Mat& image, cv::Mat& outImage,
|
||||
const cv::Size& targetShape,
|
||||
|
||||
@@ -1610,7 +1610,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath);
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1626,7 +1626,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_unique<ONNXYOLO>(_modelFilePath, engineType);
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath, engineType);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1638,26 +1638,35 @@ namespace ANSCENTER {
|
||||
// ========================================================================
|
||||
// WarmUpEngine — run a dummy inference after session creation.
|
||||
//
|
||||
// Scope: **NVIDIA (CUDA EP) only.** On first inference, the CUDA EP
|
||||
// allocates its memory arena (capped at 2 GB via BasicOrtHandler config),
|
||||
// resolves cuDNN convolution algorithms, and populates the kernel launch
|
||||
// cache. Running one dummy inference at load time amortises this cost
|
||||
// so the first real frame doesn't see a latency spike.
|
||||
// Scope: NVIDIA (CUDA EP) AND AMD (DirectML EP).
|
||||
// • NVIDIA: amortises CUDA arena allocation, cuDNN convolution algo
|
||||
// selection, and kernel-launch cache population.
|
||||
// • AMD: amortises D3D12 PSO compilation and first-time DML kernel
|
||||
// binding. Empirically ~150 ms on DirectML (see comment in
|
||||
// ONNXOCRRecognizer::Warmup). Without this, that compile cost
|
||||
// surfaces inside the first live Run() — which on AMD has been
|
||||
// observed to race with concurrent DML traffic from other sessions
|
||||
// (e.g. helmet detector running while ALPR's _lpDetector fires its
|
||||
// first ever Run()) and trigger DmlGraphFusionHelper crashes /
|
||||
// 887A0006 DEVICE_HUNG. Moving the heavy first-run work into
|
||||
// LoadEngine — where only the loading thread is active — sidesteps
|
||||
// this timing window.
|
||||
//
|
||||
// Explicitly disabled on AMD, Intel and CPU:
|
||||
// • AMD (DirectML) — calling detect() at load time has been observed
|
||||
// to hit a multi-terabyte cv::Mat allocation inside postprocessLegacy
|
||||
// on AMD RDNA iGPUs when DirectML returns garbage output tensor
|
||||
// dims. ONNXYOLO::detect() now has an output-shape sanity guard
|
||||
// that catches this at runtime, so the warm-up would add risk
|
||||
// without benefit. Earlier builds enabled warm-up specifically for
|
||||
// Radeon 680M TDR mitigation; that workaround is obsolete with
|
||||
// current DirectML 1.15.x drivers.
|
||||
// • Intel (OpenVINO) — running detect() at load time has been
|
||||
// observed to expose latent heap-corruption bugs
|
||||
// (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
|
||||
// • CPU EP — no shader compile or kernel cache to warm up; the first
|
||||
// real frame has the same latency as any subsequent frame.
|
||||
// Skipped for:
|
||||
// • Intel (OpenVINO EP) — OpenVINO compiles the full plan at session
|
||||
// creation, so first Run() has no extra compile cost. Calling
|
||||
// detect() at load time has also exposed latent heap-corruption
|
||||
// bugs (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374).
|
||||
// • CPU EP — no shader / kernel cache to warm up.
|
||||
//
|
||||
// History note: AMD warm-up was disabled in earlier builds after
|
||||
// observing multi-terabyte cv::Mat allocations from garbage DirectML
|
||||
// output dims. ONNXYOLO::detect() now has an output-shape sanity
|
||||
// guard (see line ~257, kMaxOutputDim = 1,000,000) that catches that
|
||||
// case before postprocessLegacy can allocate, so re-enabling AMD
|
||||
// warm-up is safe. The exception-swallowing catch blocks below
|
||||
// guarantee that any warm-up failure is non-fatal — the session is
|
||||
// already loaded and real inference will still work.
|
||||
//
|
||||
// Non-fatal on failure: if warm-up itself throws, regular inference
|
||||
// still works — the engine is fully loaded before WarmUpEngine runs.
|
||||
@@ -1665,9 +1674,11 @@ namespace ANSCENTER {
|
||||
void ANSONNXYOLO::WarmUpEngine() {
|
||||
if (!m_ortEngine) return;
|
||||
|
||||
// Gate strictly on NVIDIA_GPU. Every other EP is a no-op.
|
||||
if (m_ortEngine->getEngineType() != EngineType::NVIDIA_GPU) {
|
||||
ANS_DBG("ONNXYOLO", "Warm-up skipped (non-NVIDIA EP)");
|
||||
// Warm up on NVIDIA and AMD (both have lazy first-Run() compile cost).
|
||||
// Skip on CPU and OpenVINO — see scope comment above.
|
||||
const auto et = m_ortEngine->getEngineType();
|
||||
if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) {
|
||||
ANS_DBG("ONNXYOLO", "Warm-up skipped (CPU/OpenVINO EP)");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1696,7 +1707,8 @@ namespace ANSCENTER {
|
||||
// preprocessing (114,114,114 ~ 128) and avoids degenerate inputs.
|
||||
cv::Mat dummy(h, w, CV_8UC3, cv::Scalar(128, 128, 128));
|
||||
|
||||
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy CUDA inference (%dx%d)", w, h);
|
||||
ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy %s inference (%dx%d)",
|
||||
(et == EngineType::NVIDIA_GPU) ? "CUDA" : "DirectML", w, h);
|
||||
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
(void)m_ortEngine->detect(dummy, _classes,
|
||||
@@ -1975,6 +1987,150 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// HandleDmlRunFailure — stay-on-DML recovery (replaces CPU-fallback).
|
||||
//
|
||||
// Called from every catch block that wraps an Ort::Session::Run on a
|
||||
// DirectML EP. The contract is documented on the member declaration
|
||||
// in ANSONNXYOLO.h — quick summary:
|
||||
//
|
||||
// * CAS on _dmlRecreateInProgress elects ONE recreate owner per
|
||||
// driver hiccup; other concurrent failures just bail.
|
||||
// * Cooldown (kDmlRecreateCooldownMs) prevents thrashing if E_FAIL
|
||||
// keeps firing back to back.
|
||||
// * IsDmlDeviceRemoved (887A0005) → CPU fallback, last resort.
|
||||
// * After kMaxConsecutiveDmlFailures recreates without an
|
||||
// intervening successful inference → CPU fallback.
|
||||
// * Otherwise: reset() + InitOrtEngine(<original DML EP>) +
|
||||
// WarmUpEngine() → fresh DML session, GPU inference resumes.
|
||||
//
|
||||
// Successful inferences reset _dmlConsecutiveFailures to 0 in
|
||||
// DetectObjects / DetectObjectsBatch, so isolated transient hiccups
|
||||
// do not accumulate toward the CPU-fallback threshold.
|
||||
// ====================================================================
|
||||
void ANSONNXYOLO::HandleDmlRunFailure(const char* context, const std::string& msg)
|
||||
{
|
||||
// Option A: a DML exception is the strongest possible signal
|
||||
// that the AMD driver can't sustain parallel inference. Trip
|
||||
// process-wide DML serial mode immediately so the NEXT
|
||||
// inference attempt (from any thread, in this DLL or another)
|
||||
// takes the global mutex and runs one-at-a-time. This is
|
||||
// idempotent — only the first caller per process actually
|
||||
// flips the flag. Done before the CAS so even the CAS losers
|
||||
// (concurrent threads catching the same fatal) contribute the
|
||||
// signal — but in practice the first one wins.
|
||||
ANSCENTER::DmlSerialState::TripFromException(context, msg.c_str());
|
||||
|
||||
// CAS — only one thread owns the recreate at a time. Other
|
||||
// concurrent DML failures just return and retry on the next call.
|
||||
bool expected = false;
|
||||
if (!_dmlRecreateInProgress.compare_exchange_strong(expected, true)) {
|
||||
ANS_DBG("ONNXYOLO", "%s: DML fatal — another thread is recreating, skip",
|
||||
context);
|
||||
return;
|
||||
}
|
||||
|
||||
// Hold the engine mutex for the entire recreate so concurrent
|
||||
// inference threads (which use a shared_ptr local copy of the
|
||||
// engine) don't see a half-built session.
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
|
||||
try {
|
||||
// Cooldown — if we just recreated within the window, the
|
||||
// driver is in a prolonged bad state; don't burn time
|
||||
// recreating again immediately. Other threads will keep
|
||||
// returning {} until either the next call past cooldown
|
||||
// succeeds or we hit the CPU-fallback threshold.
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto sinceLastMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
now - _lastDmlRecreateAt).count();
|
||||
if (_lastDmlRecreateAt.time_since_epoch().count() != 0
|
||||
&& sinceLastMs < kDmlRecreateCooldownMs) {
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"%s: DML fatal — within cooldown (%lld ms < %lld), skip",
|
||||
context, (long long)sinceLastMs, (long long)kDmlRecreateCooldownMs);
|
||||
_dmlRecreateInProgress.store(false);
|
||||
return;
|
||||
}
|
||||
_lastDmlRecreateAt = now;
|
||||
|
||||
// Decide target EP for the new session:
|
||||
// * 887A0005 device-removed → CPU is the only option
|
||||
// * Too many consecutive failures → CPU as last resort
|
||||
// * Otherwise → recreate on the SAME DML EP we were on
|
||||
ANSCENTER::EngineType originalEp = ANSCENTER::EngineType::CPU;
|
||||
if (m_ortEngine) {
|
||||
originalEp = m_ortEngine->getEngineType();
|
||||
}
|
||||
|
||||
const int failures = _dmlConsecutiveFailures.fetch_add(1) + 1;
|
||||
const bool deviceRemoved = ANSCENTER::IsDmlDeviceRemoved(msg);
|
||||
const bool exceededRetries = failures > kMaxConsecutiveDmlFailures;
|
||||
|
||||
ANSCENTER::EngineType targetEp = originalEp;
|
||||
const char* reason = "stay-on-DML retry";
|
||||
if (deviceRemoved) {
|
||||
targetEp = ANSCENTER::EngineType::CPU;
|
||||
reason = "device-removed (887A0005) → CPU fallback";
|
||||
} else if (exceededRetries) {
|
||||
targetEp = ANSCENTER::EngineType::CPU;
|
||||
reason = "exceeded retry budget → CPU fallback";
|
||||
}
|
||||
|
||||
_logger.LogFatal(context,
|
||||
std::string("DirectML fatal — ") + reason
|
||||
+ " (failure #" + std::to_string(failures)
|
||||
+ "). msg=" + msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"%s: DML fatal recreate — %s, failures=%d, originalEp=%d, targetEp=%d",
|
||||
context, reason, failures, (int)originalEp, (int)targetEp);
|
||||
|
||||
// reset() releases the member's reference; concurrent inference
|
||||
// threads holding a shared_ptr local copy keep their old
|
||||
// engine alive until they return — no use-after-free (B2).
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(targetEp)) {
|
||||
_logger.LogFatal(context,
|
||||
"DML recreate failed — m_ortEngine left null", __FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "%s: InitOrtEngine(%d) failed", context, (int)targetEp);
|
||||
_dmlRecreateInProgress.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Warm up the new session so the first real call doesn't pay
|
||||
// the algorithm-selection / shader-compile tax. AMD DML
|
||||
// recreate WITHOUT warmup typically hits 1-2 s on the very
|
||||
// first inference, which can re-trigger our 2 s slow watchdog
|
||||
// and produce a false alarm.
|
||||
try {
|
||||
WarmUpEngine();
|
||||
} catch (const std::exception& we) {
|
||||
ANS_DBG("ONNXYOLO", "%s: WarmUpEngine after recreate threw: %s",
|
||||
context, we.what());
|
||||
}
|
||||
|
||||
// If we landed on CPU as a last resort, leave the failure
|
||||
// counter pinned so we don't oscillate back to DML on the
|
||||
// next exception. Otherwise (we're back on DML) DON'T reset
|
||||
// the counter here — it gets reset by a *successful*
|
||||
// inference in DetectObjects / DetectObjectsBatch.
|
||||
if (targetEp == ANSCENTER::EngineType::CPU
|
||||
&& originalEp != ANSCENTER::EngineType::CPU) {
|
||||
_logger.LogInfo(context,
|
||||
"Permanently switched to CPU EP — DML retry budget exhausted "
|
||||
"or device removed. Restart the process to retry GPU.",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
catch (const std::exception& re) {
|
||||
_logger.LogFatal(context,
|
||||
std::string("DML recreate exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
_dmlRecreateInProgress.store(false);
|
||||
}
|
||||
|
||||
std::vector<Object> ANSONNXYOLO::DetectObjects(const cv::Mat& inputImage,
|
||||
const std::string& camera_id)
|
||||
{
|
||||
@@ -1985,16 +2141,27 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
|
||||
// ── Snapshot config under a brief lock ──────────────────
|
||||
// ── Snapshot config + engine handle under a brief lock ──
|
||||
// Only hold _mutex long enough to validate state and copy
|
||||
// the parameters needed for inference. The actual ORT
|
||||
// detect() call runs outside the lock so that concurrent
|
||||
// Initialize/LoadModel calls are not blocked for the full
|
||||
// duration of inference.
|
||||
//
|
||||
// B2: We also snapshot a shared_ptr COPY of m_ortEngine
|
||||
// here. The lock-free detect() call below dereferences
|
||||
// the LOCAL copy (`engine`), not the member. If a
|
||||
// concurrent thread catches a DML fatal and does
|
||||
// m_ortEngine.reset() under the same lock, that only
|
||||
// releases the member's reference — our local copy keeps
|
||||
// the engine alive until this function returns. This
|
||||
// closes the use-after-free race that produced the
|
||||
// 0xc0000005 crash at ANSODEngine.dll +0x111fd0.
|
||||
float probThresh, nmsThresh;
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
{
|
||||
ANS_DBG("ONNXYOLO", "DetectObjects: cam=%s acquiring mutex...", camera_id.c_str());
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjects");
|
||||
@@ -2007,13 +2174,14 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Snapshot parameters while locked
|
||||
// Snapshot parameters + engine handle while locked
|
||||
probThresh = PROBABILITY_THRESHOLD;
|
||||
nmsThresh = NMS_THRESHOLD;
|
||||
numKps = NUM_KPS;
|
||||
classes = _classes;
|
||||
trackerEnabled = _trackerEnabled;
|
||||
stabilizationEnabled = _stabilizationEnabled;
|
||||
engine = m_ortEngine; // ref-count++
|
||||
}
|
||||
// ── _mutex released — heavy work below runs lock-free ───
|
||||
|
||||
@@ -2047,11 +2215,28 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Run ORT inference — no mutex held, this is the expensive call
|
||||
auto results = m_ortEngine->detect(inferenceImage, classes,
|
||||
probThresh,
|
||||
nmsThresh,
|
||||
numKps);
|
||||
// Run ORT inference — no engine mutex held, this is the
|
||||
// expensive call.
|
||||
// B2: dereference the LOCAL shared_ptr `engine`, not the
|
||||
// member m_ortEngine. A concurrent recreate thread may
|
||||
// reset() m_ortEngine while we're inside detect() — the
|
||||
// local ref-count keeps the underlying ONNXYOLO alive
|
||||
// until this function returns.
|
||||
//
|
||||
// Option A: DmlInferenceGuard wraps the call to (a)
|
||||
// serialize on the global DML mutex IF DML serial mode has
|
||||
// been tripped, and (b) trip serial mode if THIS call's
|
||||
// latency exceeds the slow threshold (1.5 s — well below
|
||||
// WDDM TDR). No-op on Intel/NVIDIA/CPU.
|
||||
std::vector<Object> results;
|
||||
{
|
||||
DmlInferenceGuard dmlGuard(engine->getEngineType(),
|
||||
"ANSONNXYOLO::DetectObjects");
|
||||
results = engine->detect(inferenceImage, classes,
|
||||
probThresh,
|
||||
nmsThresh,
|
||||
numKps);
|
||||
}
|
||||
|
||||
// --- Rescale coordinates from full-res back to display-res ---
|
||||
if (bgrScaleX != 1.0f || bgrScaleY != 1.0f) {
|
||||
@@ -2075,42 +2260,38 @@ namespace ANSCENTER {
|
||||
for (auto& obj : results)
|
||||
obj.cameraId = camera_id;
|
||||
|
||||
// Tracking/stabilization (ApplyTracking has its own lock)
|
||||
if (trackerEnabled && !m_ortEngine->lastWasClassification) {
|
||||
// Tracking/stabilization (ApplyTracking has its own lock).
|
||||
// B2: read lastWasClassification from the LOCAL engine copy
|
||||
// — using m_ortEngine here would re-introduce the race we
|
||||
// just closed for detect().
|
||||
if (trackerEnabled && !engine->lastWasClassification) {
|
||||
results = ApplyTracking(results, camera_id);
|
||||
if (stabilizationEnabled) results = StabilizeDetections(results, camera_id);
|
||||
}
|
||||
|
||||
// A successful inference clears the DML failure budget so
|
||||
// isolated transient hiccups (one E_FAIL every few minutes)
|
||||
// never accumulate toward kMaxConsecutiveDmlFailures and
|
||||
// wrongly demote us to CPU. Only sustained back-to-back
|
||||
// failures with no successful inference between them
|
||||
// exhaust the budget.
|
||||
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
|
||||
return results;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "DML device lost — recreating session on CPU");
|
||||
try {
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
m_ortEngine.reset();
|
||||
if (InitOrtEngine(ANSCENTER::EngineType::CPU)) {
|
||||
_logger.LogInfo("ANSONNXYOLO::DetectObjects",
|
||||
"CPU fallback session created successfully",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ONNXYOLO", "CPU fallback OK");
|
||||
} else {
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
"CPU fallback session creation failed",
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
} catch (const std::exception& re) {
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjects",
|
||||
std::string("CPU fallback exception: ") + re.what(),
|
||||
__FILE__, __LINE__);
|
||||
}
|
||||
}
|
||||
// Any DML-fatal HRESULT or DML branded error → rebuild on CPU.
|
||||
// See IsDmlFatalError() in ANSEngineCommon.h for the full set
|
||||
// (887A0005 device-removed, 80004005 generic E_FAIL from
|
||||
// DmlGraphFusionHelper / DmlCommandRecorder, etc.).
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure for full
|
||||
// contract. Tries to rebuild the session on the same
|
||||
// DML EP first; only falls back to CPU on
|
||||
// device-removed (887A0005) or after the retry budget
|
||||
// is exhausted.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::DetectObjects", msg);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -2157,19 +2338,9 @@ namespace ANSCENTER {
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
try {
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
"CPU fallback session creation failed", __FILE__, __LINE__);
|
||||
} catch (...) {}
|
||||
}
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::RunInferencesBatch", msg);
|
||||
return {};
|
||||
}
|
||||
_logger.LogFatal("ANSONNXYOLO::RunInferencesBatch",
|
||||
@@ -2184,11 +2355,15 @@ namespace ANSCENTER {
|
||||
try {
|
||||
if (_modelLoading.load()) return {};
|
||||
|
||||
// Snapshot config under brief lock
|
||||
// Snapshot config + engine handle under brief lock.
|
||||
// B2: shared_ptr copy `engine` keeps the underlying ONNXYOLO
|
||||
// alive across the lock-free detectBatch() call even if a
|
||||
// concurrent DML-fatal handler resets m_ortEngine.
|
||||
float probThresh, nmsThresh;
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
{
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjectsBatch");
|
||||
if (!lk.owns_lock()) return {};
|
||||
@@ -2204,13 +2379,20 @@ namespace ANSCENTER {
|
||||
classes = _classes;
|
||||
trackerEnabled = _trackerEnabled;
|
||||
stabilizationEnabled = _stabilizationEnabled;
|
||||
engine = m_ortEngine; // ref-count++
|
||||
}
|
||||
|
||||
// Heavy work outside lock
|
||||
auto batchResults = m_ortEngine->detectBatch(
|
||||
inputImages, classes, probThresh, nmsThresh, numKps);
|
||||
// Heavy work outside lock — dereference local copy, not member.
|
||||
// Option A: same DmlInferenceGuard contract as DetectObjects.
|
||||
std::vector<std::vector<Object>> batchResults;
|
||||
{
|
||||
DmlInferenceGuard dmlGuard(engine->getEngineType(),
|
||||
"ANSONNXYOLO::DetectObjectsBatch");
|
||||
batchResults = engine->detectBatch(
|
||||
inputImages, classes, probThresh, nmsThresh, numKps);
|
||||
}
|
||||
|
||||
const bool isClassification = m_ortEngine->lastBatchWasClassification;
|
||||
const bool isClassification = engine->lastBatchWasClassification;
|
||||
|
||||
for (auto& results : batchResults) {
|
||||
for (auto& obj : results)
|
||||
@@ -2222,24 +2404,16 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Clear DML failure budget on success — see DetectObjects
|
||||
// for the full rationale.
|
||||
_dmlConsecutiveFailures.store(0, std::memory_order_relaxed);
|
||||
return batchResults;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
"DirectML GPU device lost (887A0005) — attempting CPU fallback",
|
||||
__FILE__, __LINE__);
|
||||
try {
|
||||
std::lock_guard<std::recursive_mutex> lk(_mutex);
|
||||
m_ortEngine.reset();
|
||||
if (!InitOrtEngine(ANSCENTER::EngineType::CPU))
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
"CPU fallback session creation failed", __FILE__, __LINE__);
|
||||
} catch (...) {}
|
||||
}
|
||||
if (IsDmlFatalError(msg)) {
|
||||
// Stay-on-DML retry — see HandleDmlRunFailure.
|
||||
HandleDmlRunFailure("ANSONNXYOLO::DetectObjectsBatch", msg);
|
||||
return {};
|
||||
}
|
||||
_logger.LogFatal("ANSONNXYOLO::DetectObjectsBatch",
|
||||
|
||||
@@ -210,14 +210,74 @@ namespace ANSCENTER {
|
||||
int NUM_KPS{ 0 };
|
||||
float KPS_THRESHOLD{ 0.5f };
|
||||
|
||||
// ONNX Runtime inference engine
|
||||
std::unique_ptr<ONNXYOLO> m_ortEngine;
|
||||
// ONNX Runtime inference engine.
|
||||
//
|
||||
// B2: shared_ptr (was unique_ptr) so concurrent inference threads
|
||||
// can grab a ref-counted local copy under the brief snapshot lock
|
||||
// and dereference it OUTSIDE the lock without racing with a
|
||||
// concurrent reset() in the DML-fatal CPU-fallback path. Without
|
||||
// this, a thread doing `m_ortEngine->detect(...)` lock-free could
|
||||
// see its underlying object freed mid-call by another thread that
|
||||
// caught a DML E_FAIL and rebuilt the session — that was the
|
||||
// 0xc0000005 access violation observed in ANSODEngine.dll under
|
||||
// AMD/DirectML when multiple AI tasks ran in parallel.
|
||||
std::shared_ptr<ONNXYOLO> m_ortEngine;
|
||||
|
||||
// DML device-lost recovery: when DirectML's GPU device is removed
|
||||
// (HRESULT 887A0005), the session is permanently broken. We detect
|
||||
// this once, attempt a CPU-fallback recreation, and suppress further
|
||||
// error-log flooding.
|
||||
bool _dmlDeviceLost{ false };
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// DML stay-on-GPU recovery state.
|
||||
//
|
||||
// AMD's DirectML EP intermittently throws 80004005 (E_FAIL) from
|
||||
// DmlGraphFusionHelper / DmlCommandRecorder when the driver
|
||||
// chokes under concurrent inference pressure. Empirically this
|
||||
// is NOT a permanent device loss — recreating the ORT session on
|
||||
// the same DML EP clears the wedged state and resumes GPU
|
||||
// inference at full speed. Only the truly-fatal 887A0005
|
||||
// (DXGI_ERROR_DEVICE_REMOVED) requires CPU fallback.
|
||||
//
|
||||
// Recovery contract (used by every Run() catch block):
|
||||
// 1. compare_exchange on _dmlRecreateInProgress elects one
|
||||
// thread as the recreate owner. Other threads return {}
|
||||
// and retry on the next call.
|
||||
// 2. The owner checks the cooldown — if we recreated in the
|
||||
// last kDmlRecreateCooldownMs ms, skip (driver is in a
|
||||
// prolonged bad state, don't thrash).
|
||||
// 3. If error is IsDmlDeviceRemoved (887A0005), or we've hit
|
||||
// kMaxConsecutiveDmlFailures recreates without a successful
|
||||
// inference in between, fall back to CPU as last resort.
|
||||
// 4. Otherwise reset() + InitOrtEngine(<original DML EP>) +
|
||||
// WarmUpEngine() to rebuild on the same GPU EP.
|
||||
// 5. On successful inference, _dmlConsecutiveFailures resets
|
||||
// to 0, so isolated transient hiccups don't accumulate
|
||||
// toward the CPU-fallback threshold.
|
||||
//
|
||||
// This replaces the old single-shot `bool _dmlDeviceLost` flag
|
||||
// which latched on the first failure and forced CPU forever —
|
||||
// user requirement is to keep AI tasks running on GPU, not to
|
||||
// trade GPU performance for stability the moment the driver
|
||||
// hiccups once.
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
std::atomic<bool> _dmlRecreateInProgress{ false };
|
||||
std::atomic<int> _dmlConsecutiveFailures{ 0 };
|
||||
std::chrono::steady_clock::time_point _lastDmlRecreateAt{};
|
||||
|
||||
// Tunables — tuned for AMD Radeon 780M observed behavior.
|
||||
// threshold: catches isolated transient AMD DML hiccups. After
|
||||
// 3 consecutive recreates without a successful inference, the
|
||||
// driver is in a prolonged bad state and CPU is the only
|
||||
// forward path.
|
||||
// cooldown: prevents recreate thrash. AMD DML session creation
|
||||
// takes 200–500 ms; if E_FAIL fires from many threads at once
|
||||
// we don't want to recreate N times in a row.
|
||||
static constexpr int kMaxConsecutiveDmlFailures = 3;
|
||||
static constexpr int64_t kDmlRecreateCooldownMs = 5000;
|
||||
|
||||
// Centralised DML-fatal recovery used by all 3 catch sites.
|
||||
// Caller must have already determined the exception is DML-related
|
||||
// (IsDmlFatalError == true). msg is the exception message used
|
||||
// both to log and to detect the device-removed sub-case. Returns
|
||||
// immediately without doing anything when another thread already
|
||||
// owns the recreate (CAS loser path).
|
||||
void HandleDmlRunFailure(const char* context, const std::string& msg);
|
||||
|
||||
// Internal detection pipeline
|
||||
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
|
||||
|
||||
@@ -384,13 +384,17 @@ namespace ANSCENTER {
|
||||
catch (const std::exception& e) {
|
||||
const std::string msg = e.what();
|
||||
|
||||
// DML device-removal detection (see ANSONNXYOLO.cpp for details)
|
||||
if (msg.find("887A0005") != std::string::npos) {
|
||||
// DML fatal-error detection (887A0005 device-removed, 80004005 generic
|
||||
// DML E_FAIL, branded DmlFusedNode / DmlCommandRecorder / etc.).
|
||||
// See ANSCENTER::IsDmlFatalError in ANSEngineCommon.h.
|
||||
if (IsDmlFatalError(msg)) {
|
||||
if (!_dmlDeviceLost) {
|
||||
_dmlDeviceLost = true;
|
||||
_logger.LogFatal("ANSOVFD::RunInference",
|
||||
"DirectML GPU device lost (887A0005) — will attempt CPU fallback on next call",
|
||||
std::string("DirectML fatal error — will attempt CPU fallback on next call. msg=")
|
||||
+ msg.substr(0, 256),
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSOVFD", "DML fatal msg=%s", msg.c_str());
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user