97 lines
4.2 KiB
C++
97 lines
4.2 KiB
C++
#pragma once
|
|||
|
|
// ============================================================================
|
||
|
|
// GpuMemoryGuard — process-wide GPU VRAM safeguard.
|
||
|
|
//
|
||
|
|
// Purpose: prevent the out-of-memory cascade that happens when many TensorRT /
|
||
|
|
// CUDA engines (FR-SCRFD, OD-YOLO, LPR, liveness, ORT-CUDA, OpenCV-CUDA) share
|
||
|
|
// one GPU and collectively exhaust VRAM. The first cudaMalloc/stream OOM
|
||
|
|
// poisons the CUDA context ("permanently corrupted after N failures"), after
|
||
|
|
// which every GPU op fails and the service appears dead until restart.
|
||
|
|
//
|
||
|
|
// Three coordinated layers (all process-wide, thread-safe, lock-free):
|
||
|
|
// (1) CanLoad(need) — admission control: refuse to load an engine on GPU
|
||
|
|
// if it won't fit while keeping a safety margin free.
|
||
|
|
// (2) UnderPressure() — pressure guard: skip a frame's GPU work when free
|
||
|
|
// VRAM is below the margin (or breaker is open).
|
||
|
|
// (3) NoteOom()/Breaker— circuit breaker: after a caught OOM, open a short
|
||
|
|
// cooldown so callers back off instead of hammering
|
||
|
|
// the fragile context (which is what corrupts it).
|
||
|
|
//
|
||
|
|
// Margin default = 768 MiB; override at runtime with env ANSCENTER_GPU_MARGIN_MB.
|
||
|
|
// All queries operate on the CURRENT CUDA device (callers cudaSetDevice first).
|
||
|
|
// ============================================================================
|
||
|
|
#include <cuda_runtime.h>
|
||
|
|
#include <atomic>
|
||
|
|
#include <chrono>
|
||
|
|
#include <cstddef>
|
||
|
|
#include <cstdint>
|
||
|
|
#include <cstdlib>
|
||
|
|
|
||
|
|
namespace ANSCENTER {
|
||
|
|
|
||
|
|
class GpuMemoryGuard {
|
||
|
|
public:
|
||
|
|
// ---- configuration ----------------------------------------------------
|
||
|
|
static size_t Margin() { return s_marginBytes.load(std::memory_order_relaxed); }
|
||
|
|
static void SetMargin(size_t b) { s_marginBytes.store(b, std::memory_order_relaxed); }
|
||
|
|
|
||
|
|
// ---- (1) admission control at load ------------------------------------
|
||
|
|
// True if 'needBytes' can be committed on the current device while keeping
|
||
|
|
// the safety margin free. If VRAM can't be queried, returns true (never
|
||
|
|
// block on a failed query — let the normal path handle it).
|
||
|
|
static bool CanLoad(size_t needBytes) {
|
||
|
|
size_t freeB = 0, totalB = 0;
|
||
|
|
if (!Query(freeB, totalB)) return true;
|
||
|
|
return freeB >= needBytes + Margin();
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- (2) pressure guard at inference ----------------------------------
|
||
|
|
// True if this frame should SKIP GPU work: free VRAM below margin, or the
|
||
|
|
// circuit breaker is currently open.
|
||
|
|
static bool UnderPressure() {
|
||
|
|
if (BreakerOpen()) return true;
|
||
|
|
size_t freeB = 0, totalB = 0;
|
||
|
|
if (!Query(freeB, totalB)) return false;
|
||
|
|
return freeB < Margin();
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- (3) OOM-safe alloc + circuit breaker -----------------------------
|
||
|
|
static bool IsOom(cudaError_t e) { return e == cudaErrorMemoryAllocation; }
|
||
|
|
|
||
|
|
// Open the breaker for a short cooldown after a (recoverable) OOM so all
|
||
|
|
// callers back off together instead of repeatedly faulting the context.
|
||
|
|
static void NoteOom() {
|
||
|
|
s_breakerUntilMs.store(NowMs() + kCooldownMs, std::memory_order_relaxed);
|
||
|
|
}
|
||
|
|
static bool BreakerOpen() {
|
||
|
|
return NowMs() < s_breakerUntilMs.load(std::memory_order_relaxed);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Lightweight current-device VRAM query. Clears the non-fatal sticky error
|
||
|
|
// that a failed query can leave on the context.
|
||
|
|
static bool Query(size_t& freeB, size_t& totalB) {
|
||
|
|
cudaError_t e = cudaMemGetInfo(&freeB, &totalB);
|
||
|
|
cudaGetLastError();
|
||
|
|
return e == cudaSuccess;
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
static int64_t NowMs() {
|
||
|
|
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||
|
|
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||
|
|
}
|
||
|
|
static size_t InitMargin() {
|
||
|
|
if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) {
|
||
|
|
long mb = std::atol(e);
|
||
|
|
if (mb > 0) return static_cast<size_t>(mb) * 1024ULL * 1024ULL;
|
||
|
|
}
|
||
|
|
return 768ULL * 1024 * 1024; // 768 MiB
|
||
|
|
}
|
||
|
|
|
||
|
|
static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM
|
||
|
|
inline static std::atomic<size_t> s_marginBytes{ InitMargin() };
|
||
|
|
inline static std::atomic<int64_t> s_breakerUntilMs{ 0 };
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace ANSCENTER
|