ANSFR: gate recognizer batch size by GPU VRAM
Batch-4 was tuned for 8 GB GPUs; its TRT workspace (~54 MiB per batch unit) crowds out the other engines on smaller cards. VramCappedBatch keeps the configured batch on 8 GB-class cards and up (threshold 7.5 GiB — 8 GB cards report ~8188 MiB) and drops to single-face batches below that: identical results, multi-face frames just run sequentially, ~160 MiB freed on a 4 GB RTX A400. Also caps the legacy 8/32 LoadEngine profile, whose .b32 engine (~1.7 GB workspace) would be fatal on small cards. Initialize() and OptimizeModel() share the same call so their profiles cannot drift. Small cards rebuild the recognizer engine once (filename loses the .b4 suffix); 8 GB+ machines keep their cached engines unchanged. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,24 @@
|
||||
#define FR_END_TIMER(name, label)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
// Batch >1 only helps when several faces arrive in one frame, but the TRT
|
||||
// workspace grows ~linearly with maxBatchSize (~54 MiB per batch unit for
|
||||
// this model). The batch-4 profile was tuned for 8 GB GPUs; on smaller
|
||||
// cards (e.g. 4 GB RTX A400) that workspace crowds out the other engines
|
||||
// sharing the card. Below the 8 GB class, fall back to single-face
|
||||
// batches — results are identical, multi-face frames just run
|
||||
// sequentially. Threshold is 7.5 GiB because 8 GB cards report slightly
|
||||
// under 8192 MiB (~8188).
|
||||
int VramCappedBatch(int desiredBatch, int deviceIndex) {
|
||||
cudaDeviceProp prop{};
|
||||
if (cudaGetDeviceProperties(&prop, deviceIndex >= 0 ? deviceIndex : 0) != cudaSuccess)
|
||||
return desiredBatch; // can't query — keep the configured profile
|
||||
const size_t totalMiB = prop.totalGlobalMem / (1024ULL * 1024ULL);
|
||||
return (totalMiB >= 7680) ? desiredBatch : 1;
|
||||
}
|
||||
}
|
||||
|
||||
namespace ANSCENTER {
|
||||
std::string ANSFaceRecognizer::GetOpenVINODevice() {
|
||||
ov::Core core;
|
||||
@@ -97,8 +115,10 @@ namespace ANSCENTER {
|
||||
// comfortably on 8 GB GPUs while freeing ~1.5 GB VRAM vs max=32
|
||||
// — most scenes have ≤4 faces visible, so throughput cost is
|
||||
// near-zero (amortized per-face latency drops too at lower batch).
|
||||
m_options.optBatchSize = 4;
|
||||
m_options.maxBatchSize = 4;
|
||||
// VramCappedBatch drops this to 1 on sub-8 GB cards.
|
||||
const int frBatch = VramCappedBatch(4, _modelConfig.gpuDeviceIndex);
|
||||
m_options.optBatchSize = frBatch;
|
||||
m_options.maxBatchSize = frBatch;
|
||||
|
||||
m_options.maxInputHeight = GPU_FACE_HEIGHT;
|
||||
m_options.minInputHeight = GPU_FACE_HEIGHT;
|
||||
@@ -259,9 +279,10 @@ namespace ANSCENTER {
|
||||
// engine built/looked-up here has a different profile than the one
|
||||
// Initialize() expects, causing either a stale `.b32` rebuild every
|
||||
// launch (wasting 30-60s) or a load-time profile mismatch.
|
||||
// Keep in sync with lines 100-114.
|
||||
m_options.optBatchSize = 4;
|
||||
m_options.maxBatchSize = 4;
|
||||
// Keep in sync with the Initialize() block (same VramCappedBatch call).
|
||||
const int frBatch = VramCappedBatch(4, _modelConfig.gpuDeviceIndex);
|
||||
m_options.optBatchSize = frBatch;
|
||||
m_options.maxBatchSize = frBatch;
|
||||
|
||||
m_options.maxInputHeight = GPU_FACE_HEIGHT;
|
||||
m_options.minInputHeight = GPU_FACE_HEIGHT;
|
||||
@@ -490,8 +511,12 @@ namespace ANSCENTER {
|
||||
if (engineType == EngineType::NVIDIA_GPU) {
|
||||
// Initialize TensorRT engine via shared pool
|
||||
if (!m_trtEngine) {
|
||||
m_options.optBatchSize = 8;
|
||||
m_options.maxBatchSize = 32;
|
||||
// Note this path still uses the legacy 8/32 profile on 8 GB+
|
||||
// cards (a .b32 engine costs ~1.5 GB more workspace than .b4).
|
||||
// The VRAM cap only stops it from being fatal on small cards.
|
||||
const int frMaxBatch = VramCappedBatch(32, _modelConfig.gpuDeviceIndex);
|
||||
m_options.optBatchSize = (frMaxBatch < 8) ? frMaxBatch : 8;
|
||||
m_options.maxBatchSize = frMaxBatch;
|
||||
m_options.calibrationBatchSize = 8;
|
||||
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
|
||||
m_options.engineFileDir = _modelFolder;
|
||||
|
||||
Reference in New Issue
Block a user