Prevent OOM
This commit is contained in:
@@ -761,6 +761,16 @@ namespace ANSCENTER
|
||||
float CosineSimilarity(const std::vector<float>& a, const std::vector<float>& b, bool normalized);
|
||||
};
|
||||
class ANSENGINE_API ANSODBase {
|
||||
public:
|
||||
// Explicit non-copy/non-move: _modelLoading (std::atomic) deletes the
|
||||
// implicit copy ops; spelling them out here gives ClangCL a clean
|
||||
// anchor and documents that subclasses are never copied/moved (they
|
||||
// are heap-allocated and passed by shared_ptr<...>).
|
||||
ANSODBase(const ANSODBase&) = delete;
|
||||
ANSODBase& operator=(const ANSODBase&) = delete;
|
||||
ANSODBase(ANSODBase&&) = delete;
|
||||
ANSODBase& operator=(ANSODBase&&) = delete;
|
||||
ANSODBase() = default;
|
||||
protected:
|
||||
bool _licenseValid{ false };
|
||||
bool _isInitialized{ false };
|
||||
@@ -1085,6 +1095,13 @@ namespace ANSCENTER
|
||||
std::vector<Object> RunInferenceInTrackingMode(const cv::Mat& input, const std::string& camera_id, std::vector<int> trackingObjectIds);// Get detected objects
|
||||
};
|
||||
class ANSENGINE_API ANSFDBase {
|
||||
public:
|
||||
// Explicit non-copy/non-move (see ANSODBase for rationale).
|
||||
ANSFDBase(const ANSFDBase&) = delete;
|
||||
ANSFDBase& operator=(const ANSFDBase&) = delete;
|
||||
ANSFDBase(ANSFDBase&&) = delete;
|
||||
ANSFDBase& operator=(ANSFDBase&&) = delete;
|
||||
ANSFDBase() = default;
|
||||
protected:
|
||||
bool _licenseValid{ false };
|
||||
bool _isInitialized{ false };
|
||||
|
||||
@@ -1316,7 +1316,7 @@ namespace ANSCENTER
|
||||
{
|
||||
try {
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(ANSCENTER::EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[ANSFDBase] EP ready: "<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ namespace ANSCENTER
|
||||
inputImageShape_ = targetInputShape;
|
||||
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[ANSONNXCL] EP ready: "
|
||||
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
@@ -25,6 +25,13 @@ namespace ANSCENTER {
|
||||
// ────────────────────────────────────────────────────────────────────────
|
||||
class ANSENGINE_API ANSONNXCL :public ANSODBase {
|
||||
public:
|
||||
// Explicit non-copy/non-move (see ANSONNXYOLO for dllexport rationale).
|
||||
ANSONNXCL(const ANSONNXCL&) = delete;
|
||||
ANSONNXCL& operator=(const ANSONNXCL&) = delete;
|
||||
ANSONNXCL(ANSONNXCL&&) = delete;
|
||||
ANSONNXCL& operator=(ANSONNXCL&&) = delete;
|
||||
ANSONNXCL() = default;
|
||||
|
||||
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
|
||||
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
|
||||
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
|
||||
|
||||
@@ -692,7 +692,7 @@ namespace ANSCENTER {
|
||||
deviceId_ = deviceId;
|
||||
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[ANSONNXOBB] EP ready: "
|
||||
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
@@ -276,7 +276,7 @@ namespace ANSCENTER {
|
||||
deviceId_ = deviceId;
|
||||
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[ANSONNXPOSE] EP ready: "
|
||||
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
@@ -108,6 +108,13 @@ namespace ANSCENTER {
|
||||
// ====================================================================
|
||||
class ANSENGINE_API ANSONNXRTMPOSE : public ANSODBase {
|
||||
public:
|
||||
// Explicit non-copy/non-move (see ANSONNXYOLO for dllexport rationale).
|
||||
ANSONNXRTMPOSE(const ANSONNXRTMPOSE&) = delete;
|
||||
ANSONNXRTMPOSE& operator=(const ANSONNXRTMPOSE&) = delete;
|
||||
ANSONNXRTMPOSE(ANSONNXRTMPOSE&&) = delete;
|
||||
ANSONNXRTMPOSE& operator=(ANSONNXRTMPOSE&&) = delete;
|
||||
ANSONNXRTMPOSE() = default;
|
||||
|
||||
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||
const std::string& modelZipFilePath,
|
||||
const std::string& modelZipPassword,
|
||||
|
||||
@@ -204,7 +204,7 @@ namespace ANSCENTER
|
||||
}
|
||||
|
||||
try {
|
||||
m_engine = std::make_unique<ONNXSAM3>(modelFolder, EngineType::NVIDIA_GPU);
|
||||
m_engine = std::make_unique<class ONNXSAM3>(modelFolder, EngineType::NVIDIA_GPU);
|
||||
m_tokenLength = m_engine->getTokenLength();
|
||||
|
||||
_logger.LogDebug("ANSONNXSAM3::InitEngine",
|
||||
|
||||
@@ -16,6 +16,13 @@ namespace ANSCENTER
|
||||
class ANSENGINE_API ANSONNXSAM3 : public ANSODBase
|
||||
{
|
||||
public:
|
||||
// Explicit non-copy/non-move (see ANSONNXYOLO for dllexport rationale).
|
||||
ANSONNXSAM3(const ANSONNXSAM3&) = delete;
|
||||
ANSONNXSAM3& operator=(const ANSONNXSAM3&) = delete;
|
||||
ANSONNXSAM3(ANSONNXSAM3&&) = delete;
|
||||
ANSONNXSAM3& operator=(ANSONNXSAM3&&) = delete;
|
||||
ANSONNXSAM3() = default;
|
||||
|
||||
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||
const std::string& modelZipFilePath, const std::string& modelZipPassword,
|
||||
std::string& labelMap) override;
|
||||
@@ -45,7 +52,10 @@ namespace ANSCENTER
|
||||
bool _fp16 = false;
|
||||
|
||||
// ONNX Runtime engine
|
||||
std::unique_ptr<ONNXSAM3> m_engine;
|
||||
// Elaborated `class ONNXSAM3` disambiguates from the
|
||||
// ANSCENTER::ModelType::ONNXSAM3 enumerator (ANSLicense.h) which
|
||||
// shares the same unqualified name in this namespace.
|
||||
std::unique_ptr<class ONNXSAM3> m_engine;
|
||||
|
||||
// Tokenizer (loaded from merges.txt)
|
||||
std::unique_ptr<ANSCLIPTokenizer> m_tokenizer;
|
||||
|
||||
@@ -273,7 +273,7 @@ namespace ANSCENTER {
|
||||
deviceId_ = deviceId;
|
||||
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[ANSONNXSEG] EP ready: "
|
||||
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
@@ -129,7 +129,7 @@ namespace ANSCENTER {
|
||||
// future ORT shape-mismatch failure (e.g. /model.12/Concat for
|
||||
// non-stride-32 inputs) points at the offending HxW directly.
|
||||
// One line per session — does not flood DebugView.
|
||||
static thread_local const ONNXYOLO* s_loggedFirst = nullptr;
|
||||
static thread_local const class ONNXYOLO* s_loggedFirst = nullptr;
|
||||
if (s_loggedFirst != this) {
|
||||
s_loggedFirst = this;
|
||||
ANS_DBG("ONNXYOLO",
|
||||
@@ -1680,7 +1680,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath);
|
||||
m_ortEngine = std::make_shared<class ONNXYOLO>(_modelFilePath);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -1696,7 +1696,7 @@ namespace ANSCENTER {
|
||||
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
|
||||
return false;
|
||||
}
|
||||
m_ortEngine = std::make_shared<ONNXYOLO>(_modelFilePath, engineType);
|
||||
m_ortEngine = std::make_shared<class ONNXYOLO>(_modelFilePath, engineType);
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
@@ -2244,7 +2244,7 @@ namespace ANSCENTER {
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
std::shared_ptr<class ONNXYOLO> engine;
|
||||
{
|
||||
ANS_DBG("ONNXYOLO", "DetectObjects: cam=%s acquiring mutex...", camera_id.c_str());
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjects");
|
||||
@@ -2446,7 +2446,7 @@ namespace ANSCENTER {
|
||||
int numKps;
|
||||
std::vector<std::string> classes;
|
||||
bool trackerEnabled, stabilizationEnabled;
|
||||
std::shared_ptr<ONNXYOLO> engine;
|
||||
std::shared_ptr<class ONNXYOLO> engine;
|
||||
{
|
||||
auto lk = TryLockWithTimeout("ANSONNXYOLO::DetectObjectsBatch");
|
||||
if (!lk.owns_lock()) return {};
|
||||
|
||||
@@ -205,6 +205,16 @@ namespace ANSCENTER {
|
||||
// ====================================================================
|
||||
class ANSENGINE_API ANSONNXYOLO : public ANSODBase {
|
||||
public:
|
||||
// Explicit non-copy/non-move. The class has atomic + shared_ptr members
|
||||
// that already implicitly delete these, but __declspec(dllexport)
|
||||
// makes ClangCL eagerly instantiate the implicit copy/move ctor
|
||||
// bodies; an explicit user-declared =delete short-circuits that.
|
||||
ANSONNXYOLO(const ANSONNXYOLO&) = delete;
|
||||
ANSONNXYOLO& operator=(const ANSONNXYOLO&) = delete;
|
||||
ANSONNXYOLO(ANSONNXYOLO&&) = delete;
|
||||
ANSONNXYOLO& operator=(ANSONNXYOLO&&) = delete;
|
||||
ANSONNXYOLO() = default;
|
||||
|
||||
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
|
||||
const std::string& modelZipFilePath,
|
||||
const std::string& modelZipPassword,
|
||||
@@ -254,7 +264,10 @@ namespace ANSCENTER {
|
||||
// caught a DML E_FAIL and rebuilt the session — that was the
|
||||
// 0xc0000005 access violation observed in ANSODEngine.dll under
|
||||
// AMD/DirectML when multiple AI tasks ran in parallel.
|
||||
std::shared_ptr<ONNXYOLO> m_ortEngine;
|
||||
// Elaborated `class ONNXYOLO` disambiguates from the
|
||||
// ANSCENTER::ModelType::ONNXYOLO enumerator (ANSLicense.h) which
|
||||
// shares the same unqualified name in this namespace.
|
||||
std::shared_ptr<class ONNXYOLO> m_ortEngine;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// DML stay-on-GPU recovery state.
|
||||
|
||||
@@ -281,7 +281,7 @@ namespace ANSCENTER {
|
||||
{
|
||||
try {
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[YOLO12OD] EP ready: "
|
||||
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
@@ -208,7 +208,7 @@ namespace ANSCENTER
|
||||
{
|
||||
try {
|
||||
const auto& ep = ANSCENTER::EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
std::cout << "[YOLOOD] EP ready: "
|
||||
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "ANSEngineCommon.h"
|
||||
#include "ANSLicense.h" // ANS_DBG macro
|
||||
#include "GpuMemoryGuard.h"
|
||||
#include <opencv2/cudaimgproc.hpp>
|
||||
#include <opencv2/cudawarping.hpp>
|
||||
#include <opencv2/core/cuda_stream_accessor.hpp>
|
||||
@@ -168,6 +169,12 @@ namespace ANSCENTER {
|
||||
}
|
||||
|
||||
++m_cudaFailStreak;
|
||||
// (3) If this failure is a GPU OOM, open the process-wide circuit
|
||||
// breaker so every engine (FR/OD/OCR/LPR) backs off via UnderPressure()
|
||||
// before the streak reaches CUDA_FAIL_LIMIT and the context is lost.
|
||||
if (GpuMemoryGuard::IsOom(err) || GpuMemoryGuard::IsOom(pending)) {
|
||||
GpuMemoryGuard::NoteOom();
|
||||
}
|
||||
if (m_cudaFailStreak >= CUDA_FAIL_LIMIT && !m_cudaContextDead) {
|
||||
m_cudaContextDead = true;
|
||||
logger.LogFatal(std::string(tag) + "::NV12Helper",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "NV12PreprocessHelper.h" // tl_currentGpuFrame()
|
||||
#include "Utility.h"
|
||||
#include "GpuMemoryGuard.h"
|
||||
#include <chrono>
|
||||
//#define FNS_DEBUG
|
||||
namespace ANSCENTER {
|
||||
@@ -638,6 +639,17 @@ namespace ANSCENTER {
|
||||
// Phase 2: CUDA preprocessing + inference (mutex released — pool dispatches to idle GPU slot)
|
||||
std::vector<std::vector<cv::cuda::GpuMat>> inputs;
|
||||
bool usedNV12 = false;
|
||||
|
||||
// (2) Pressure guard: if free GPU VRAM is below the safety margin (or the
|
||||
// OOM circuit breaker is open), SKIP this frame's GPU work instead of
|
||||
// allocating and triggering an OOM that permanently poisons the context.
|
||||
if (ANSCENTER::GpuMemoryGuard::UnderPressure()) {
|
||||
this->_logger.LogWarn("ANSSCRFDFD::Detect",
|
||||
"GPU under memory pressure — skipping SCRFD frame to avoid OOM",
|
||||
__FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
// Clear any sticky CUDA error from transient graph-capture failures
|
||||
cudaError_t priorErr = cudaGetLastError();
|
||||
@@ -706,6 +718,14 @@ namespace ANSCENTER {
|
||||
m_nv12Helper.tickInference();
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
// (3) On a GPU OOM, open the circuit breaker so subsequent frames
|
||||
// back off (UnderPressure) instead of repeatedly faulting the
|
||||
// already-fragile CUDA context — that repetition is what corrupts it.
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("out of memory") != std::string::npos ||
|
||||
msg.find("Gpu API call") != std::string::npos) {
|
||||
ANSCENTER::GpuMemoryGuard::NoteOom();
|
||||
}
|
||||
this->_logger.LogError(
|
||||
"ANSSCRFDFD::Detect",
|
||||
std::string("CUDA preprocessing failed: ") + e.what(),
|
||||
|
||||
Reference in New Issue
Block a user