GpuMemoryGuard: scale VRAM safety margin to card size
The margin was a fixed 768 MiB — 3% of a 24 GiB compute card but 19% of a 4 GiB card (RTX A400), where free VRAM never clears it, so CanLoad() refused every engine and UnderPressure() skipped every frame. Now total/8 clamped to [192, 768] MiB: cards >= 6 GiB keep exactly the old margin, only small cards change. ANSCENTER_GPU_MARGIN_MB / SetMargin() still override (0 = auto). Refusal log now reports total VRAM alongside free, which is what made this diagnosable. Verified on the A400 test PC: detector + recognizer load and match. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -818,9 +818,10 @@ bool Engine<T>::loadNetwork(std::string trtModelPath, const std::array<float, 3>
|
|||||||
std::cerr << "[GpuMemGuard] REFUSING engine load on GPU "
|
std::cerr << "[GpuMemGuard] REFUSING engine load on GPU "
|
||||||
<< m_options.deviceIndex << ": need ~"
|
<< m_options.deviceIndex << ": need ~"
|
||||||
<< (buffer.size() / (1024 * 1024)) << " MiB + "
|
<< (buffer.size() / (1024 * 1024)) << " MiB + "
|
||||||
<< (ANSCENTER::GpuMemoryGuard::Margin() / (1024 * 1024))
|
<< (ANSCENTER::GpuMemoryGuard::MarginFor(totalVRAM) / (1024 * 1024))
|
||||||
<< " MiB margin, only " << (freeVRAM / (1024 * 1024))
|
<< " MiB margin, only " << (freeVRAM / (1024 * 1024))
|
||||||
<< " MiB free. Reduce concurrent engines/cameras or batch size."
|
<< " MiB free of " << (totalVRAM / (1024 * 1024))
|
||||||
|
<< " MiB total. Reduce concurrent engines/cameras or batch size."
|
||||||
<< std::endl;
|
<< std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
+37
-10
@@ -17,8 +17,15 @@
|
|||||||
// cooldown so callers back off instead of hammering
|
// cooldown so callers back off instead of hammering
|
||||||
// the fragile context (which is what corrupts it).
|
// the fragile context (which is what corrupts it).
|
||||||
//
|
//
|
||||||
// Margin default = 768 MiB; override at runtime with env ANSCENTER_GPU_MARGIN_MB.
|
// The margin scales with the device: total/8, clamped to [192 MiB, 768 MiB].
|
||||||
// All queries operate on the CURRENT CUDA device (callers cudaSetDevice first).
|
// A fixed margin cannot work across the fleet — 768 MiB is 3% of a 24 GiB
|
||||||
|
// compute card but 19% of a 4 GiB display card (RTX A400), where the desktop
|
||||||
|
// plus the ORT CUDA arena never leave that much free and admission control can
|
||||||
|
// therefore never pass. The 768 MiB ceiling keeps every card >= 6 GiB on
|
||||||
|
// exactly the previous margin, so only small cards change behaviour.
|
||||||
|
// Override at runtime with env ANSCENTER_GPU_MARGIN_MB (or SetMargin); an
|
||||||
|
// explicit override wins over scaling. All queries operate on the CURRENT CUDA
|
||||||
|
// device (callers cudaSetDevice first).
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
#include <cuda_runtime.h>
|
#include <cuda_runtime.h>
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
@@ -32,8 +39,24 @@ namespace ANSCENTER {
|
|||||||
class GpuMemoryGuard {
|
class GpuMemoryGuard {
|
||||||
public:
|
public:
|
||||||
// ---- configuration ----------------------------------------------------
|
// ---- configuration ----------------------------------------------------
|
||||||
static size_t Margin() { return s_marginBytes.load(std::memory_order_relaxed); }
|
// Effective margin for a device with 'totalB' VRAM. An explicit override
|
||||||
static void SetMargin(size_t b) { s_marginBytes.store(b, std::memory_order_relaxed); }
|
// (env or SetMargin) wins; otherwise scale to the card.
|
||||||
|
static size_t MarginFor(size_t totalB) {
|
||||||
|
if (size_t ov = s_marginOverride.load(std::memory_order_relaxed)) return ov;
|
||||||
|
size_t m = totalB / kMarginDivisor;
|
||||||
|
if (m < kMarginMin) m = kMarginMin;
|
||||||
|
if (m > kMarginMax) m = kMarginMax;
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
// Effective margin for the CURRENT device. Prefer MarginFor() where the
|
||||||
|
// caller already has a total from Query() — this re-queries.
|
||||||
|
static size_t Margin() {
|
||||||
|
size_t freeB = 0, totalB = 0;
|
||||||
|
if (!Query(freeB, totalB)) return kMarginMax;
|
||||||
|
return MarginFor(totalB);
|
||||||
|
}
|
||||||
|
// b == 0 restores device-scaled margins.
|
||||||
|
static void SetMargin(size_t b) { s_marginOverride.store(b, std::memory_order_relaxed); }
|
||||||
|
|
||||||
// ---- (1) admission control at load ------------------------------------
|
// ---- (1) admission control at load ------------------------------------
|
||||||
// True if 'needBytes' can be committed on the current device while keeping
|
// True if 'needBytes' can be committed on the current device while keeping
|
||||||
@@ -42,7 +65,7 @@ public:
|
|||||||
static bool CanLoad(size_t needBytes) {
|
static bool CanLoad(size_t needBytes) {
|
||||||
size_t freeB = 0, totalB = 0;
|
size_t freeB = 0, totalB = 0;
|
||||||
if (!Query(freeB, totalB)) return true;
|
if (!Query(freeB, totalB)) return true;
|
||||||
return freeB >= needBytes + Margin();
|
return freeB >= needBytes + MarginFor(totalB);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- (2) pressure guard at inference ----------------------------------
|
// ---- (2) pressure guard at inference ----------------------------------
|
||||||
@@ -52,7 +75,7 @@ public:
|
|||||||
if (BreakerOpen()) return true;
|
if (BreakerOpen()) return true;
|
||||||
size_t freeB = 0, totalB = 0;
|
size_t freeB = 0, totalB = 0;
|
||||||
if (!Query(freeB, totalB)) return false;
|
if (!Query(freeB, totalB)) return false;
|
||||||
return freeB < Margin();
|
return freeB < MarginFor(totalB);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- (3) OOM-safe alloc + circuit breaker -----------------------------
|
// ---- (3) OOM-safe alloc + circuit breaker -----------------------------
|
||||||
@@ -80,16 +103,20 @@ private:
|
|||||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||||
std::chrono::steady_clock::now().time_since_epoch()).count();
|
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||||
}
|
}
|
||||||
static size_t InitMargin() {
|
// 0 = no override, use device-scaled margin.
|
||||||
|
static size_t InitMarginOverride() {
|
||||||
if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) {
|
if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) {
|
||||||
long mb = std::atol(e);
|
long mb = std::atol(e);
|
||||||
if (mb > 0) return static_cast<size_t>(mb) * 1024ULL * 1024ULL;
|
if (mb > 0) return static_cast<size_t>(mb) * 1024ULL * 1024ULL;
|
||||||
}
|
}
|
||||||
return 768ULL * 1024 * 1024; // 768 MiB
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM
|
static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM
|
||||||
inline static std::atomic<size_t> s_marginBytes{ InitMargin() };
|
static constexpr size_t kMarginDivisor = 8; // margin = total/8 ...
|
||||||
|
static constexpr size_t kMarginMin = 192ULL * 1024 * 1024; // ... clamped
|
||||||
|
static constexpr size_t kMarginMax = 768ULL * 1024 * 1024; // (prev. fixed value)
|
||||||
|
inline static std::atomic<size_t> s_marginOverride{ InitMarginOverride() };
|
||||||
inline static std::atomic<int64_t> s_breakerUntilMs{ 0 };
|
inline static std::atomic<int64_t> s_breakerUntilMs{ 0 };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user