GpuMemoryGuard: scale VRAM safety margin to card size

The margin was a fixed 768 MiB — 3% of a 24 GiB compute card but 19% of
a 4 GiB card (RTX A400), where free VRAM never clears it, so CanLoad()
refused every engine and UnderPressure() skipped every frame. Now
total/8 clamped to [192, 768] MiB: cards >= 6 GiB keep exactly the old
margin, only small cards change. ANSCENTER_GPU_MARGIN_MB / SetMargin()
still override (0 = auto). Refusal log now reports total VRAM alongside
free, which is what made this diagnosable.

Verified on the A400 test PC: detector + recognizer load and match.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-17 13:10:10 +10:00
co-authored by Claude Opus 4.8
parent ac79e51d17
commit d2479b9b07
2 changed files with 40 additions and 12 deletions
@@ -818,9 +818,10 @@ bool Engine<T>::loadNetwork(std::string trtModelPath, const std::array<float, 3>
std::cerr << "[GpuMemGuard] REFUSING engine load on GPU " std::cerr << "[GpuMemGuard] REFUSING engine load on GPU "
<< m_options.deviceIndex << ": need ~" << m_options.deviceIndex << ": need ~"
<< (buffer.size() / (1024 * 1024)) << " MiB + " << (buffer.size() / (1024 * 1024)) << " MiB + "
<< (ANSCENTER::GpuMemoryGuard::Margin() / (1024 * 1024)) << (ANSCENTER::GpuMemoryGuard::MarginFor(totalVRAM) / (1024 * 1024))
<< " MiB margin, only " << (freeVRAM / (1024 * 1024)) << " MiB margin, only " << (freeVRAM / (1024 * 1024))
<< " MiB free. Reduce concurrent engines/cameras or batch size." << " MiB free of " << (totalVRAM / (1024 * 1024))
<< " MiB total. Reduce concurrent engines/cameras or batch size."
<< std::endl; << std::endl;
return false; return false;
} }
+36 -9
View File
@@ -17,8 +17,15 @@
// cooldown so callers back off instead of hammering // cooldown so callers back off instead of hammering
// the fragile context (which is what corrupts it). // the fragile context (which is what corrupts it).
// //
// Margin default = 768 MiB; override at runtime with env ANSCENTER_GPU_MARGIN_MB. // The margin scales with the device: total/8, clamped to [192 MiB, 768 MiB].
// All queries operate on the CURRENT CUDA device (callers cudaSetDevice first). // A fixed margin cannot work across the fleet — 768 MiB is 3% of a 24 GiB
// compute card but 19% of a 4 GiB display card (RTX A400), where the desktop
// plus the ORT CUDA arena never leave that much free and admission control can
// therefore never pass. The 768 MiB ceiling keeps every card >= 6 GiB on
// exactly the previous margin, so only small cards change behaviour.
// Override at runtime with env ANSCENTER_GPU_MARGIN_MB (or SetMargin); an
// explicit override wins over scaling. All queries operate on the CURRENT CUDA
// device (callers cudaSetDevice first).
// ============================================================================ // ============================================================================
#include <cuda_runtime.h> #include <cuda_runtime.h>
#include <atomic> #include <atomic>
@@ -32,8 +39,24 @@ namespace ANSCENTER {
class GpuMemoryGuard { class GpuMemoryGuard {
public: public:
// ---- configuration ---------------------------------------------------- // ---- configuration ----------------------------------------------------
static size_t Margin() { return s_marginBytes.load(std::memory_order_relaxed); } // Effective margin for a device with 'totalB' VRAM. An explicit override
static void SetMargin(size_t b) { s_marginBytes.store(b, std::memory_order_relaxed); } // (env or SetMargin) wins; otherwise scale to the card.
static size_t MarginFor(size_t totalB) {
if (size_t ov = s_marginOverride.load(std::memory_order_relaxed)) return ov;
size_t m = totalB / kMarginDivisor;
if (m < kMarginMin) m = kMarginMin;
if (m > kMarginMax) m = kMarginMax;
return m;
}
// Effective margin for the CURRENT device. Prefer MarginFor() where the
// caller already has a total from Query() — this re-queries.
static size_t Margin() {
size_t freeB = 0, totalB = 0;
if (!Query(freeB, totalB)) return kMarginMax;
return MarginFor(totalB);
}
// b == 0 restores device-scaled margins.
static void SetMargin(size_t b) { s_marginOverride.store(b, std::memory_order_relaxed); }
// ---- (1) admission control at load ------------------------------------ // ---- (1) admission control at load ------------------------------------
// True if 'needBytes' can be committed on the current device while keeping // True if 'needBytes' can be committed on the current device while keeping
@@ -42,7 +65,7 @@ public:
static bool CanLoad(size_t needBytes) { static bool CanLoad(size_t needBytes) {
size_t freeB = 0, totalB = 0; size_t freeB = 0, totalB = 0;
if (!Query(freeB, totalB)) return true; if (!Query(freeB, totalB)) return true;
return freeB >= needBytes + Margin(); return freeB >= needBytes + MarginFor(totalB);
} }
// ---- (2) pressure guard at inference ---------------------------------- // ---- (2) pressure guard at inference ----------------------------------
@@ -52,7 +75,7 @@ public:
if (BreakerOpen()) return true; if (BreakerOpen()) return true;
size_t freeB = 0, totalB = 0; size_t freeB = 0, totalB = 0;
if (!Query(freeB, totalB)) return false; if (!Query(freeB, totalB)) return false;
return freeB < Margin(); return freeB < MarginFor(totalB);
} }
// ---- (3) OOM-safe alloc + circuit breaker ----------------------------- // ---- (3) OOM-safe alloc + circuit breaker -----------------------------
@@ -80,16 +103,20 @@ private:
return std::chrono::duration_cast<std::chrono::milliseconds>( return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count(); std::chrono::steady_clock::now().time_since_epoch()).count();
} }
static size_t InitMargin() { // 0 = no override, use device-scaled margin.
static size_t InitMarginOverride() {
if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) { if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) {
long mb = std::atol(e); long mb = std::atol(e);
if (mb > 0) return static_cast<size_t>(mb) * 1024ULL * 1024ULL; if (mb > 0) return static_cast<size_t>(mb) * 1024ULL * 1024ULL;
} }
return 768ULL * 1024 * 1024; // 768 MiB return 0;
} }
static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM
inline static std::atomic<size_t> s_marginBytes{ InitMargin() }; static constexpr size_t kMarginDivisor = 8; // margin = total/8 ...
static constexpr size_t kMarginMin = 192ULL * 1024 * 1024; // ... clamped
static constexpr size_t kMarginMax = 768ULL * 1024 * 1024; // (prev. fixed value)
inline static std::atomic<size_t> s_marginOverride{ InitMarginOverride() };
inline static std::atomic<int64_t> s_breakerUntilMs{ 0 }; inline static std::atomic<int64_t> s_breakerUntilMs{ 0 };
}; };