Prevent OOM
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "NV12PreprocessHelper.h" // tl_currentGpuFrame()
|
||||
#include "Utility.h"
|
||||
#include "GpuMemoryGuard.h"
|
||||
#include <chrono>
|
||||
//#define FNS_DEBUG
|
||||
namespace ANSCENTER {
|
||||
@@ -638,6 +639,17 @@ namespace ANSCENTER {
|
||||
// Phase 2: CUDA preprocessing + inference (mutex released — pool dispatches to idle GPU slot)
|
||||
std::vector<std::vector<cv::cuda::GpuMat>> inputs;
|
||||
bool usedNV12 = false;
|
||||
|
||||
// (2) Pressure guard: if free GPU VRAM is below the safety margin (or the
|
||||
// OOM circuit breaker is open), SKIP this frame's GPU work instead of
|
||||
// allocating and triggering an OOM that permanently poisons the context.
|
||||
if (ANSCENTER::GpuMemoryGuard::UnderPressure()) {
|
||||
this->_logger.LogWarn("ANSSCRFDFD::Detect",
|
||||
"GPU under memory pressure — skipping SCRFD frame to avoid OOM",
|
||||
__FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
// Clear any sticky CUDA error from transient graph-capture failures
|
||||
cudaError_t priorErr = cudaGetLastError();
|
||||
@@ -706,6 +718,14 @@ namespace ANSCENTER {
|
||||
m_nv12Helper.tickInference();
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
// (3) On a GPU OOM, open the circuit breaker so subsequent frames
|
||||
// back off (UnderPressure) instead of repeatedly faulting the
|
||||
// already-fragile CUDA context — that repetition is what corrupts it.
|
||||
const std::string msg = e.what();
|
||||
if (msg.find("out of memory") != std::string::npos ||
|
||||
msg.find("Gpu API call") != std::string::npos) {
|
||||
ANSCENTER::GpuMemoryGuard::NoteOom();
|
||||
}
|
||||
this->_logger.LogError(
|
||||
"ANSSCRFDFD::Detect",
|
||||
std::string("CUDA preprocessing failed: ") + e.what(),
|
||||
|
||||
Reference in New Issue
Block a user