Pool dispatch: prefer slot on the GPU where inputs already reside

Two-pass slot acquisition in runInferenceFromPool: Pass A picks an idle
slot whose deviceIndex matches the input GpuMats' resident device
(zero-copy dispatch); Pass B falls back to plain round-robin, where
runInference migrates the inputs via the pinned-host two-hop copy.

Callers that prepare frames on GPU A no longer pay the ~1 ms/frame PCIe
migration when a same-GPU slot is idle, while spillover to the other GPU
under load still works. Input residency is queried outside the slot
mutex via cudaPointerGetAttributes (cheap driver lookup).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-06 20:05:00 +10:00
co-authored by Claude Fable 5
parent a4ea532f36
commit 13180921c9
@@ -606,6 +606,25 @@ bool Engine<T>::runInferenceFromPool(
bool kickedGrowth = false; bool kickedGrowth = false;
auto _poolAcquireStart = std::chrono::steady_clock::now(); auto _poolAcquireStart = std::chrono::steady_clock::now();
// -- DEVICE AFFINITY: find which GPU the caller's inputs live on ------
// Dispatching to a slot on the same GPU avoids the pinned-host migration
// copy (~1 ms per 1080p frame) that runInference performs for foreign
// inputs. Queried outside the slot mutex — it's a cheap driver lookup.
int inputDevice = -1;
for (const auto& batch : inputs) {
for (const auto& img : batch) {
if (!img.data) continue;
cudaPointerAttributes attr{};
if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess &&
attr.type == cudaMemoryTypeDevice) {
inputDevice = attr.device;
}
cudaGetLastError(); // clear any error the query left behind
break; // first image is representative
}
if (inputDevice >= 0) break;
}
{ {
std::unique_lock<std::mutex> lock(m_slotMutex); std::unique_lock<std::mutex> lock(m_slotMutex);
@@ -616,13 +635,31 @@ bool Engine<T>::runInferenceFromPool(
const size_t n = m_slots.size(); const size_t n = m_slots.size();
if (n > 0) { if (n > 0) {
const size_t start = m_nextSlotHint.load() % n; const size_t start = m_nextSlotHint.load() % n;
for (size_t i = 0; i < n; ++i) {
auto& s = m_slots[(start + i) % n]; // Pass A: idle slot on the inputs' own GPU (zero-copy dispatch)
if (!s.busy && s.engine) { // alive and idle if (inputDevice >= 0) {
s.busy = true; for (size_t i = 0; i < n; ++i) {
slot = &s; auto& s = m_slots[(start + i) % n];
m_nextSlotHint = (start + i + 1) % n; if (!s.busy && s.engine && s.deviceIndex == inputDevice) {
break; s.busy = true;
slot = &s;
m_nextSlotHint = (start + i + 1) % n;
break;
}
}
}
// Pass B: any idle slot (round-robin) — runInference migrates
// the inputs across GPUs when needed.
if (!slot) {
for (size_t i = 0; i < n; ++i) {
auto& s = m_slots[(start + i) % n];
if (!s.busy && s.engine) { // alive and idle
s.busy = true;
slot = &s;
m_nextSlotHint = (start + i + 1) % n;
break;
}
} }
} }
} }