Pool dispatch: prefer slot on the GPU where inputs already reside
Two-pass slot acquisition in runInferenceFromPool: Pass A picks an idle slot whose deviceIndex matches the input GpuMats' resident device (zero-copy dispatch); Pass B falls back to plain round-robin, where runInference migrates the inputs via the pinned-host two-hop copy. Callers that prepare frames on GPU A no longer pay the ~1 ms/frame PCIe migration when a same-GPU slot is idle, while spillover to the other GPU under load still works. Input residency is queried outside the slot mutex via cudaPointerGetAttributes (cheap driver lookup). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -606,6 +606,25 @@ bool Engine<T>::runInferenceFromPool(
|
|||||||
bool kickedGrowth = false;
|
bool kickedGrowth = false;
|
||||||
auto _poolAcquireStart = std::chrono::steady_clock::now();
|
auto _poolAcquireStart = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
|
// -- DEVICE AFFINITY: find which GPU the caller's inputs live on ------
|
||||||
|
// Dispatching to a slot on the same GPU avoids the pinned-host migration
|
||||||
|
// copy (~1 ms per 1080p frame) that runInference performs for foreign
|
||||||
|
// inputs. Queried outside the slot mutex — it's a cheap driver lookup.
|
||||||
|
int inputDevice = -1;
|
||||||
|
for (const auto& batch : inputs) {
|
||||||
|
for (const auto& img : batch) {
|
||||||
|
if (!img.data) continue;
|
||||||
|
cudaPointerAttributes attr{};
|
||||||
|
if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess &&
|
||||||
|
attr.type == cudaMemoryTypeDevice) {
|
||||||
|
inputDevice = attr.device;
|
||||||
|
}
|
||||||
|
cudaGetLastError(); // clear any error the query left behind
|
||||||
|
break; // first image is representative
|
||||||
|
}
|
||||||
|
if (inputDevice >= 0) break;
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(m_slotMutex);
|
std::unique_lock<std::mutex> lock(m_slotMutex);
|
||||||
|
|
||||||
@@ -616,13 +635,31 @@ bool Engine<T>::runInferenceFromPool(
|
|||||||
const size_t n = m_slots.size();
|
const size_t n = m_slots.size();
|
||||||
if (n > 0) {
|
if (n > 0) {
|
||||||
const size_t start = m_nextSlotHint.load() % n;
|
const size_t start = m_nextSlotHint.load() % n;
|
||||||
for (size_t i = 0; i < n; ++i) {
|
|
||||||
auto& s = m_slots[(start + i) % n];
|
// Pass A: idle slot on the inputs' own GPU (zero-copy dispatch)
|
||||||
if (!s.busy && s.engine) { // alive and idle
|
if (inputDevice >= 0) {
|
||||||
s.busy = true;
|
for (size_t i = 0; i < n; ++i) {
|
||||||
slot = &s;
|
auto& s = m_slots[(start + i) % n];
|
||||||
m_nextSlotHint = (start + i + 1) % n;
|
if (!s.busy && s.engine && s.deviceIndex == inputDevice) {
|
||||||
break;
|
s.busy = true;
|
||||||
|
slot = &s;
|
||||||
|
m_nextSlotHint = (start + i + 1) % n;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pass B: any idle slot (round-robin) — runInference migrates
|
||||||
|
// the inputs across GPUs when needed.
|
||||||
|
if (!slot) {
|
||||||
|
for (size_t i = 0; i < n; ++i) {
|
||||||
|
auto& s = m_slots[(start + i) % n];
|
||||||
|
if (!s.busy && s.engine) { // alive and idle
|
||||||
|
s.busy = true;
|
||||||
|
slot = &s;
|
||||||
|
m_nextSlotHint = (start + i + 1) % n;
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user