ANSFR: gate recognizer batch size by GPU VRAM

Batch-4 was tuned for 8 GB GPUs; its TRT workspace (~54 MiB per batch
unit) crowds out the other engines on smaller cards. VramCappedBatch
keeps the configured batch on 8 GB-class cards and up (threshold
7.5 GiB — 8 GB cards report ~8188 MiB) and drops to single-face batches
below that: identical results, multi-face frames just run sequentially,
~160 MiB freed on a 4 GB RTX A400. Also caps the legacy 8/32 LoadEngine
profile, whose .b32 engine (~1.7 GB workspace) would be fatal on small
cards. Initialize() and OptimizeModel() share the same call so their
profiles cannot drift.

Small cards rebuild the recognizer engine once (filename loses the .b4
suffix); 8 GB+ machines keep their cached engines unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-08-21 10:44:22 +10:00
co-authored by Claude Opus 4.8
parent 6939775960
commit da9d922498
+32 -7
View File
@@ -17,6 +17,24 @@
#define FR_END_TIMER(name, label)
#endif
namespace {
// Batch >1 only helps when several faces arrive in one frame, but the TRT
// workspace grows ~linearly with maxBatchSize (~54 MiB per batch unit for
// this model). The batch-4 profile was tuned for 8 GB GPUs; on smaller
// cards (e.g. 4 GB RTX A400) that workspace crowds out the other engines
// sharing the card. Below the 8 GB class, fall back to single-face
// batches — results are identical, multi-face frames just run
// sequentially. Threshold is 7.5 GiB because 8 GB cards report slightly
// under 8192 MiB (~8188).
int VramCappedBatch(int desiredBatch, int deviceIndex) {
cudaDeviceProp prop{};
if (cudaGetDeviceProperties(&prop, deviceIndex >= 0 ? deviceIndex : 0) != cudaSuccess)
return desiredBatch; // can't query — keep the configured profile
const size_t totalMiB = prop.totalGlobalMem / (1024ULL * 1024ULL);
return (totalMiB >= 7680) ? desiredBatch : 1;
}
}
namespace ANSCENTER {
std::string ANSFaceRecognizer::GetOpenVINODevice() {
ov::Core core;
@@ -97,8 +115,10 @@ namespace ANSCENTER {
// comfortably on 8 GB GPUs while freeing ~1.5 GB VRAM vs max=32
// — most scenes have ≤4 faces visible, so throughput cost is
// near-zero (amortized per-face latency drops too at lower batch).
m_options.optBatchSize = 4;
m_options.maxBatchSize = 4;
// VramCappedBatch drops this to 1 on sub-8 GB cards.
const int frBatch = VramCappedBatch(4, _modelConfig.gpuDeviceIndex);
m_options.optBatchSize = frBatch;
m_options.maxBatchSize = frBatch;
m_options.maxInputHeight = GPU_FACE_HEIGHT;
m_options.minInputHeight = GPU_FACE_HEIGHT;
@@ -259,9 +279,10 @@ namespace ANSCENTER {
// engine built/looked-up here has a different profile than the one
// Initialize() expects, causing either a stale `.b32` rebuild every
// launch (wasting 30-60s) or a load-time profile mismatch.
// Keep in sync with lines 100-114.
m_options.optBatchSize = 4;
m_options.maxBatchSize = 4;
// Keep in sync with the Initialize() block (same VramCappedBatch call).
const int frBatch = VramCappedBatch(4, _modelConfig.gpuDeviceIndex);
m_options.optBatchSize = frBatch;
m_options.maxBatchSize = frBatch;
m_options.maxInputHeight = GPU_FACE_HEIGHT;
m_options.minInputHeight = GPU_FACE_HEIGHT;
@@ -490,8 +511,12 @@ namespace ANSCENTER {
if (engineType == EngineType::NVIDIA_GPU) {
// Initialize TensorRT engine via shared pool
if (!m_trtEngine) {
m_options.optBatchSize = 8;
m_options.maxBatchSize = 32;
// Note this path still uses the legacy 8/32 profile on 8 GB+
// cards (a .b32 engine costs ~1.5 GB more workspace than .b4).
// The VRAM cap only stops it from being fatal on small cards.
const int frMaxBatch = VramCappedBatch(32, _modelConfig.gpuDeviceIndex);
m_options.optBatchSize = (frMaxBatch < 8) ? frMaxBatch : 8;
m_options.maxBatchSize = frMaxBatch;
m_options.calibrationBatchSize = 8;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.engineFileDir = _modelFolder;