Prevent OOM

This commit is contained in:
2026-06-18 05:55:31 +10:00
parent ebba70b9d9
commit b3436a61ee
44 changed files with 1797 additions and 66 deletions
+20
View File
@@ -2,6 +2,7 @@
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h" // tl_currentGpuFrame()
#include "Utility.h"
#include "GpuMemoryGuard.h"
#include <chrono>
//#define FNS_DEBUG
namespace ANSCENTER {
@@ -638,6 +639,17 @@ namespace ANSCENTER {
// Phase 2: CUDA preprocessing + inference (mutex released — pool dispatches to idle GPU slot)
std::vector<std::vector<cv::cuda::GpuMat>> inputs;
bool usedNV12 = false;
// (2) Pressure guard: if free GPU VRAM is below the safety margin (or the
// OOM circuit breaker is open), SKIP this frame's GPU work instead of
// allocating and triggering an OOM that permanently poisons the context.
if (ANSCENTER::GpuMemoryGuard::UnderPressure()) {
this->_logger.LogWarn("ANSSCRFDFD::Detect",
"GPU under memory pressure — skipping SCRFD frame to avoid OOM",
__FILE__, __LINE__);
return {};
}
try {
// Clear any sticky CUDA error from transient graph-capture failures
cudaError_t priorErr = cudaGetLastError();
@@ -706,6 +718,14 @@ namespace ANSCENTER {
m_nv12Helper.tickInference();
}
catch (const std::exception& e) {
// (3) On a GPU OOM, open the circuit breaker so subsequent frames
// back off (UnderPressure) instead of repeatedly faulting the
// already-fragile CUDA context — that repetition is what corrupts it.
const std::string msg = e.what();
if (msg.find("out of memory") != std::string::npos ||
msg.find("Gpu API call") != std::string::npos) {
ANSCENTER::GpuMemoryGuard::NoteOom();
}
this->_logger.LogError(
"ANSSCRFDFD::Detect",
std::string("CUDA preprocessing failed: ") + e.what(),