#pragma once // ============================================================================ // GpuMemoryGuard — process-wide GPU VRAM safeguard. // // Purpose: prevent the out-of-memory cascade that happens when many TensorRT / // CUDA engines (FR-SCRFD, OD-YOLO, LPR, liveness, ORT-CUDA, OpenCV-CUDA) share // one GPU and collectively exhaust VRAM. The first cudaMalloc/stream OOM // poisons the CUDA context ("permanently corrupted after N failures"), after // which every GPU op fails and the service appears dead until restart. // // Three coordinated layers (all process-wide, thread-safe, lock-free): // (1) CanLoad(need) — admission control: refuse to load an engine on GPU // if it won't fit while keeping a safety margin free. // (2) UnderPressure() — pressure guard: skip a frame's GPU work when free // VRAM is below the margin (or breaker is open). // (3) NoteOom()/Breaker— circuit breaker: after a caught OOM, open a short // cooldown so callers back off instead of hammering // the fragile context (which is what corrupts it). // // The margin scales with the device: total/8, clamped to [192 MiB, 768 MiB]. // A fixed margin cannot work across the fleet — 768 MiB is 3% of a 24 GiB // compute card but 19% of a 4 GiB display card (RTX A400), where the desktop // plus the ORT CUDA arena never leave that much free and admission control can // therefore never pass. The 768 MiB ceiling keeps every card >= 6 GiB on // exactly the previous margin, so only small cards change behaviour. // Override at runtime with env ANSCENTER_GPU_MARGIN_MB (or SetMargin); an // explicit override wins over scaling. All queries operate on the CURRENT CUDA // device (callers cudaSetDevice first). // ============================================================================ #include #include #include #include #include #include namespace ANSCENTER { class GpuMemoryGuard { public: // ---- configuration ---------------------------------------------------- // Effective margin for a device with 'totalB' VRAM. An explicit override // (env or SetMargin) wins; otherwise scale to the card. static size_t MarginFor(size_t totalB) { if (size_t ov = s_marginOverride.load(std::memory_order_relaxed)) return ov; size_t m = totalB / kMarginDivisor; if (m < kMarginMin) m = kMarginMin; if (m > kMarginMax) m = kMarginMax; return m; } // Effective margin for the CURRENT device. Prefer MarginFor() where the // caller already has a total from Query() — this re-queries. static size_t Margin() { size_t freeB = 0, totalB = 0; if (!Query(freeB, totalB)) return kMarginMax; return MarginFor(totalB); } // b == 0 restores device-scaled margins. static void SetMargin(size_t b) { s_marginOverride.store(b, std::memory_order_relaxed); } // ---- (1) admission control at load ------------------------------------ // True if 'needBytes' can be committed on the current device while keeping // the safety margin free. If VRAM can't be queried, returns true (never // block on a failed query — let the normal path handle it). static bool CanLoad(size_t needBytes) { size_t freeB = 0, totalB = 0; if (!Query(freeB, totalB)) return true; return freeB >= needBytes + MarginFor(totalB); } // ---- (2) pressure guard at inference ---------------------------------- // True if this frame should SKIP GPU work: free VRAM below margin, or the // circuit breaker is currently open. static bool UnderPressure() { if (BreakerOpen()) return true; size_t freeB = 0, totalB = 0; if (!Query(freeB, totalB)) return false; return freeB < MarginFor(totalB); } // ---- (3) OOM-safe alloc + circuit breaker ----------------------------- static bool IsOom(cudaError_t e) { return e == cudaErrorMemoryAllocation; } // Open the breaker for a short cooldown after a (recoverable) OOM so all // callers back off together instead of repeatedly faulting the context. static void NoteOom() { s_breakerUntilMs.store(NowMs() + kCooldownMs, std::memory_order_relaxed); } static bool BreakerOpen() { return NowMs() < s_breakerUntilMs.load(std::memory_order_relaxed); } // Lightweight current-device VRAM query. Clears the non-fatal sticky error // that a failed query can leave on the context. static bool Query(size_t& freeB, size_t& totalB) { cudaError_t e = cudaMemGetInfo(&freeB, &totalB); cudaGetLastError(); return e == cudaSuccess; } private: static int64_t NowMs() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()).count(); } // 0 = no override, use device-scaled margin. static size_t InitMarginOverride() { if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) { long mb = std::atol(e); if (mb > 0) return static_cast(mb) * 1024ULL * 1024ULL; } return 0; } static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM static constexpr size_t kMarginDivisor = 8; // margin = total/8 ... static constexpr size_t kMarginMin = 192ULL * 1024 * 1024; // ... clamped static constexpr size_t kMarginMax = 768ULL * 1024 * 1024; // (prev. fixed value) inline static std::atomic s_marginOverride{ InitMarginOverride() }; inline static std::atomic s_breakerUntilMs{ 0 }; }; } // namespace ANSCENTER