Prevent OOM
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
#include <sstream>
|
||||
#include "Utility.h"
|
||||
#include "TRTCompat.h"
|
||||
#include "GpuMemoryGuard.h"
|
||||
|
||||
// ============================================================================
|
||||
// Crash-safe wrappers for TensorRT operations that can crash the process.
|
||||
@@ -805,13 +806,22 @@ bool Engine<T>::loadNetwork(std::string trtModelPath, const std::array<float, 3>
|
||||
// (cudaMalloc failure during inference) when too many tasks are loaded.
|
||||
{
|
||||
size_t freeVRAM = 0, totalVRAM = 0;
|
||||
cudaError_t memErr = cudaMemGetInfo(&freeVRAM, &totalVRAM);
|
||||
constexpr size_t kMinFreeBytes = 256ULL * 1024 * 1024; // 256 MiB minimum
|
||||
if (memErr != cudaSuccess) {
|
||||
// cudaMemGetInfo failed — CUDA context may not be initialized on this thread.
|
||||
// Don't reject: let TRT try to deserialize (it may succeed).
|
||||
} else if (freeVRAM < kMinFreeBytes) {
|
||||
m_lastLoadFailedVRAM = true; // signal to buildLoadNetwork: engine file is NOT corrupt
|
||||
bool haveMem = ANSCENTER::GpuMemoryGuard::Query(freeVRAM, totalVRAM);
|
||||
// (1) Admission control: refuse to load this engine on the GPU if it
|
||||
// does not fit while keeping the GpuMemoryGuard safety margin free.
|
||||
// Prevents VRAM over-subscription across concurrent engines (FR/OD/LPR/
|
||||
// liveness) that would otherwise OOM and permanently poison the CUDA
|
||||
// context. The engine file is valid — m_lastLoadFailedVRAM tells the
|
||||
// caller to preserve it (no rebuild) and just skip GPU load for now.
|
||||
if (haveMem && !ANSCENTER::GpuMemoryGuard::CanLoad(buffer.size())) {
|
||||
m_lastLoadFailedVRAM = true;
|
||||
std::cerr << "[GpuMemGuard] REFUSING engine load on GPU "
|
||||
<< m_options.deviceIndex << ": need ~"
|
||||
<< (buffer.size() / (1024 * 1024)) << " MiB + "
|
||||
<< (ANSCENTER::GpuMemoryGuard::Margin() / (1024 * 1024))
|
||||
<< " MiB margin, only " << (freeVRAM / (1024 * 1024))
|
||||
<< " MiB free. Reduce concurrent engines/cameras or batch size."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
if (m_verbose) {
|
||||
|
||||
Reference in New Issue
Block a user