diff --git a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl index 16ac459..62a5d0e 100644 --- a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl +++ b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl @@ -818,9 +818,10 @@ bool Engine::loadNetwork(std::string trtModelPath, const std::array std::cerr << "[GpuMemGuard] REFUSING engine load on GPU " << m_options.deviceIndex << ": need ~" << (buffer.size() / (1024 * 1024)) << " MiB + " - << (ANSCENTER::GpuMemoryGuard::Margin() / (1024 * 1024)) + << (ANSCENTER::GpuMemoryGuard::MarginFor(totalVRAM) / (1024 * 1024)) << " MiB margin, only " << (freeVRAM / (1024 * 1024)) - << " MiB free. Reduce concurrent engines/cameras or batch size." + << " MiB free of " << (totalVRAM / (1024 * 1024)) + << " MiB total. Reduce concurrent engines/cameras or batch size." << std::endl; return false; } diff --git a/include/GpuMemoryGuard.h b/include/GpuMemoryGuard.h index fd2cbae..9d6ecbd 100644 --- a/include/GpuMemoryGuard.h +++ b/include/GpuMemoryGuard.h @@ -17,8 +17,15 @@ // cooldown so callers back off instead of hammering // the fragile context (which is what corrupts it). // -// Margin default = 768 MiB; override at runtime with env ANSCENTER_GPU_MARGIN_MB. -// All queries operate on the CURRENT CUDA device (callers cudaSetDevice first). +// The margin scales with the device: total/8, clamped to [192 MiB, 768 MiB]. +// A fixed margin cannot work across the fleet — 768 MiB is 3% of a 24 GiB +// compute card but 19% of a 4 GiB display card (RTX A400), where the desktop +// plus the ORT CUDA arena never leave that much free and admission control can +// therefore never pass. The 768 MiB ceiling keeps every card >= 6 GiB on +// exactly the previous margin, so only small cards change behaviour. +// Override at runtime with env ANSCENTER_GPU_MARGIN_MB (or SetMargin); an +// explicit override wins over scaling. All queries operate on the CURRENT CUDA +// device (callers cudaSetDevice first). // ============================================================================ #include #include @@ -32,8 +39,24 @@ namespace ANSCENTER { class GpuMemoryGuard { public: // ---- configuration ---------------------------------------------------- - static size_t Margin() { return s_marginBytes.load(std::memory_order_relaxed); } - static void SetMargin(size_t b) { s_marginBytes.store(b, std::memory_order_relaxed); } + // Effective margin for a device with 'totalB' VRAM. An explicit override + // (env or SetMargin) wins; otherwise scale to the card. + static size_t MarginFor(size_t totalB) { + if (size_t ov = s_marginOverride.load(std::memory_order_relaxed)) return ov; + size_t m = totalB / kMarginDivisor; + if (m < kMarginMin) m = kMarginMin; + if (m > kMarginMax) m = kMarginMax; + return m; + } + // Effective margin for the CURRENT device. Prefer MarginFor() where the + // caller already has a total from Query() — this re-queries. + static size_t Margin() { + size_t freeB = 0, totalB = 0; + if (!Query(freeB, totalB)) return kMarginMax; + return MarginFor(totalB); + } + // b == 0 restores device-scaled margins. + static void SetMargin(size_t b) { s_marginOverride.store(b, std::memory_order_relaxed); } // ---- (1) admission control at load ------------------------------------ // True if 'needBytes' can be committed on the current device while keeping @@ -42,7 +65,7 @@ public: static bool CanLoad(size_t needBytes) { size_t freeB = 0, totalB = 0; if (!Query(freeB, totalB)) return true; - return freeB >= needBytes + Margin(); + return freeB >= needBytes + MarginFor(totalB); } // ---- (2) pressure guard at inference ---------------------------------- @@ -52,7 +75,7 @@ public: if (BreakerOpen()) return true; size_t freeB = 0, totalB = 0; if (!Query(freeB, totalB)) return false; - return freeB < Margin(); + return freeB < MarginFor(totalB); } // ---- (3) OOM-safe alloc + circuit breaker ----------------------------- @@ -80,16 +103,20 @@ private: return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()).count(); } - static size_t InitMargin() { + // 0 = no override, use device-scaled margin. + static size_t InitMarginOverride() { if (const char* e = std::getenv("ANSCENTER_GPU_MARGIN_MB")) { long mb = std::atol(e); if (mb > 0) return static_cast(mb) * 1024ULL * 1024ULL; } - return 768ULL * 1024 * 1024; // 768 MiB + return 0; } - static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM - inline static std::atomic s_marginBytes{ InitMargin() }; + static constexpr int64_t kCooldownMs = 1500; // breaker cooldown after OOM + static constexpr size_t kMarginDivisor = 8; // margin = total/8 ... + static constexpr size_t kMarginMin = 192ULL * 1024 * 1024; // ... clamped + static constexpr size_t kMarginMax = 768ULL * 1024 * 1024; // (prev. fixed value) + inline static std::atomic s_marginOverride{ InitMarginOverride() }; inline static std::atomic s_breakerUntilMs{ 0 }; };