From 13180921c904bd493b31412f7f0600b23dfec3de Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Mon, 6 Jul 2026 20:05:00 +1000 Subject: [PATCH] Pool dispatch: prefer slot on the GPU where inputs already reside Two-pass slot acquisition in runInferenceFromPool: Pass A picks an idle slot whose deviceIndex matches the input GpuMats' resident device (zero-copy dispatch); Pass B falls back to plain round-robin, where runInference migrates the inputs via the pinned-host two-hop copy. Callers that prepare frames on GPU A no longer pay the ~1 ms/frame PCIe migration when a same-GPU slot is idle, while spillover to the other GPU under load still works. Input residency is queried outside the slot mutex via cudaPointerGetAttributes (cheap driver lookup). Co-Authored-By: Claude Fable 5 --- .../include/engine/EngineMultiGpu.inl | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/engines/TensorRTAPI/include/engine/EngineMultiGpu.inl b/engines/TensorRTAPI/include/engine/EngineMultiGpu.inl index 37915d2..c60b470 100644 --- a/engines/TensorRTAPI/include/engine/EngineMultiGpu.inl +++ b/engines/TensorRTAPI/include/engine/EngineMultiGpu.inl @@ -606,6 +606,25 @@ bool Engine::runInferenceFromPool( bool kickedGrowth = false; auto _poolAcquireStart = std::chrono::steady_clock::now(); + // -- DEVICE AFFINITY: find which GPU the caller's inputs live on ------ + // Dispatching to a slot on the same GPU avoids the pinned-host migration + // copy (~1 ms per 1080p frame) that runInference performs for foreign + // inputs. Queried outside the slot mutex — it's a cheap driver lookup. + int inputDevice = -1; + for (const auto& batch : inputs) { + for (const auto& img : batch) { + if (!img.data) continue; + cudaPointerAttributes attr{}; + if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess && + attr.type == cudaMemoryTypeDevice) { + inputDevice = attr.device; + } + cudaGetLastError(); // clear any error the query left behind + break; // first image is representative + } + if (inputDevice >= 0) break; + } + { std::unique_lock lock(m_slotMutex); @@ -616,13 +635,31 @@ bool Engine::runInferenceFromPool( const size_t n = m_slots.size(); if (n > 0) { const size_t start = m_nextSlotHint.load() % n; - for (size_t i = 0; i < n; ++i) { - auto& s = m_slots[(start + i) % n]; - if (!s.busy && s.engine) { // alive and idle - s.busy = true; - slot = &s; - m_nextSlotHint = (start + i + 1) % n; - break; + + // Pass A: idle slot on the inputs' own GPU (zero-copy dispatch) + if (inputDevice >= 0) { + for (size_t i = 0; i < n; ++i) { + auto& s = m_slots[(start + i) % n]; + if (!s.busy && s.engine && s.deviceIndex == inputDevice) { + s.busy = true; + slot = &s; + m_nextSlotHint = (start + i + 1) % n; + break; + } + } + } + + // Pass B: any idle slot (round-robin) — runInference migrates + // the inputs across GPUs when needed. + if (!slot) { + for (size_t i = 0; i < n; ++i) { + auto& s = m_slots[(start + i) % n]; + if (!s.busy && s.engine) { // alive and idle + s.busy = true; + slot = &s; + m_nextSlotHint = (start + i + 1) % n; + break; + } } } }