Pool dispatch: prefer slot on the GPU where inputs already reside
Two-pass slot acquisition in runInferenceFromPool: Pass A picks an idle slot whose deviceIndex matches the input GpuMats' resident device (zero-copy dispatch); Pass B falls back to plain round-robin, where runInference migrates the inputs via the pinned-host two-hop copy. Callers that prepare frames on GPU A no longer pay the ~1 ms/frame PCIe migration when a same-GPU slot is idle, while spillover to the other GPU under load still works. Input residency is queried outside the slot mutex via cudaPointerGetAttributes (cheap driver lookup). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -606,6 +606,25 @@ bool Engine<T>::runInferenceFromPool(
|
||||
bool kickedGrowth = false;
|
||||
auto _poolAcquireStart = std::chrono::steady_clock::now();
|
||||
|
||||
// -- DEVICE AFFINITY: find which GPU the caller's inputs live on ------
|
||||
// Dispatching to a slot on the same GPU avoids the pinned-host migration
|
||||
// copy (~1 ms per 1080p frame) that runInference performs for foreign
|
||||
// inputs. Queried outside the slot mutex — it's a cheap driver lookup.
|
||||
int inputDevice = -1;
|
||||
for (const auto& batch : inputs) {
|
||||
for (const auto& img : batch) {
|
||||
if (!img.data) continue;
|
||||
cudaPointerAttributes attr{};
|
||||
if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess &&
|
||||
attr.type == cudaMemoryTypeDevice) {
|
||||
inputDevice = attr.device;
|
||||
}
|
||||
cudaGetLastError(); // clear any error the query left behind
|
||||
break; // first image is representative
|
||||
}
|
||||
if (inputDevice >= 0) break;
|
||||
}
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_slotMutex);
|
||||
|
||||
@@ -616,13 +635,31 @@ bool Engine<T>::runInferenceFromPool(
|
||||
const size_t n = m_slots.size();
|
||||
if (n > 0) {
|
||||
const size_t start = m_nextSlotHint.load() % n;
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
auto& s = m_slots[(start + i) % n];
|
||||
if (!s.busy && s.engine) { // alive and idle
|
||||
s.busy = true;
|
||||
slot = &s;
|
||||
m_nextSlotHint = (start + i + 1) % n;
|
||||
break;
|
||||
|
||||
// Pass A: idle slot on the inputs' own GPU (zero-copy dispatch)
|
||||
if (inputDevice >= 0) {
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
auto& s = m_slots[(start + i) % n];
|
||||
if (!s.busy && s.engine && s.deviceIndex == inputDevice) {
|
||||
s.busy = true;
|
||||
slot = &s;
|
||||
m_nextSlotHint = (start + i + 1) % n;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass B: any idle slot (round-robin) — runInference migrates
|
||||
// the inputs across GPUs when needed.
|
||||
if (!slot) {
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
auto& s = m_slots[(start + i) % n];
|
||||
if (!s.busy && s.engine) { // alive and idle
|
||||
s.busy = true;
|
||||
slot = &s;
|
||||
m_nextSlotHint = (start + i + 1) % n;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user