From da9d922498e0d1f885b2658800eb8bb336e3b2a8 Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Fri, 21 Aug 2026 10:44:22 +1000 Subject: [PATCH] ANSFR: gate recognizer batch size by GPU VRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Batch-4 was tuned for 8 GB GPUs; its TRT workspace (~54 MiB per batch unit) crowds out the other engines on smaller cards. VramCappedBatch keeps the configured batch on 8 GB-class cards and up (threshold 7.5 GiB — 8 GB cards report ~8188 MiB) and drops to single-face batches below that: identical results, multi-face frames just run sequentially, ~160 MiB freed on a 4 GB RTX A400. Also caps the legacy 8/32 LoadEngine profile, whose .b32 engine (~1.7 GB workspace) would be fatal on small cards. Initialize() and OptimizeModel() share the same call so their profiles cannot drift. Small cards rebuild the recognizer engine once (filename loses the .b4 suffix); 8 GB+ machines keep their cached engines unchanged. Co-Authored-By: Claude Opus 4.8 --- modules/ANSFR/ANSFaceRecognizer.cpp | 39 +++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/modules/ANSFR/ANSFaceRecognizer.cpp b/modules/ANSFR/ANSFaceRecognizer.cpp index 2f60604..9eb570f 100644 --- a/modules/ANSFR/ANSFaceRecognizer.cpp +++ b/modules/ANSFR/ANSFaceRecognizer.cpp @@ -17,6 +17,24 @@ #define FR_END_TIMER(name, label) #endif +namespace { + // Batch >1 only helps when several faces arrive in one frame, but the TRT + // workspace grows ~linearly with maxBatchSize (~54 MiB per batch unit for + // this model). The batch-4 profile was tuned for 8 GB GPUs; on smaller + // cards (e.g. 4 GB RTX A400) that workspace crowds out the other engines + // sharing the card. Below the 8 GB class, fall back to single-face + // batches — results are identical, multi-face frames just run + // sequentially. Threshold is 7.5 GiB because 8 GB cards report slightly + // under 8192 MiB (~8188). + int VramCappedBatch(int desiredBatch, int deviceIndex) { + cudaDeviceProp prop{}; + if (cudaGetDeviceProperties(&prop, deviceIndex >= 0 ? deviceIndex : 0) != cudaSuccess) + return desiredBatch; // can't query — keep the configured profile + const size_t totalMiB = prop.totalGlobalMem / (1024ULL * 1024ULL); + return (totalMiB >= 7680) ? desiredBatch : 1; + } +} + namespace ANSCENTER { std::string ANSFaceRecognizer::GetOpenVINODevice() { ov::Core core; @@ -97,8 +115,10 @@ namespace ANSCENTER { // comfortably on 8 GB GPUs while freeing ~1.5 GB VRAM vs max=32 // — most scenes have ≤4 faces visible, so throughput cost is // near-zero (amortized per-face latency drops too at lower batch). - m_options.optBatchSize = 4; - m_options.maxBatchSize = 4; + // VramCappedBatch drops this to 1 on sub-8 GB cards. + const int frBatch = VramCappedBatch(4, _modelConfig.gpuDeviceIndex); + m_options.optBatchSize = frBatch; + m_options.maxBatchSize = frBatch; m_options.maxInputHeight = GPU_FACE_HEIGHT; m_options.minInputHeight = GPU_FACE_HEIGHT; @@ -259,9 +279,10 @@ namespace ANSCENTER { // engine built/looked-up here has a different profile than the one // Initialize() expects, causing either a stale `.b32` rebuild every // launch (wasting 30-60s) or a load-time profile mismatch. - // Keep in sync with lines 100-114. - m_options.optBatchSize = 4; - m_options.maxBatchSize = 4; + // Keep in sync with the Initialize() block (same VramCappedBatch call). + const int frBatch = VramCappedBatch(4, _modelConfig.gpuDeviceIndex); + m_options.optBatchSize = frBatch; + m_options.maxBatchSize = frBatch; m_options.maxInputHeight = GPU_FACE_HEIGHT; m_options.minInputHeight = GPU_FACE_HEIGHT; @@ -490,8 +511,12 @@ namespace ANSCENTER { if (engineType == EngineType::NVIDIA_GPU) { // Initialize TensorRT engine via shared pool if (!m_trtEngine) { - m_options.optBatchSize = 8; - m_options.maxBatchSize = 32; + // Note this path still uses the legacy 8/32 profile on 8 GB+ + // cards (a .b32 engine costs ~1.5 GB more workspace than .b4). + // The VRAM cap only stops it from being fatal on small cards. + const int frMaxBatch = VramCappedBatch(32, _modelConfig.gpuDeviceIndex); + m_options.optBatchSize = (frMaxBatch < 8) ? frMaxBatch : 8; + m_options.maxBatchSize = frMaxBatch; m_options.calibrationBatchSize = 8; m_options.deviceIndex = _modelConfig.gpuDeviceIndex; m_options.engineFileDir = _modelFolder;