Fix multi-GPU inference on Blackwell (dual RTX 5080): four stacked fixes

Verified on dual RTX 5080: 355/355 detections, both GPUs in round-robin,
zero CUDA errors. Requires TensorRT >= 10.16.1 in SharedCore (10.15.1 has
a Myelin l2cm crash when creating a context on a second Blackwell device).

1. blobFromGpuMats: thread_local GpuMat caches (tl_blob/tl_floatImg) were
   device-blind — a pool thread warming GPU[1] then GPU[0] reused device-1
   memory with a device-0 stream, causing an async illegal memory access
   that poisoned every CUDA context (sticky 700). Track owning device and
   rebuild the cache on device switch.

2. runInference: pool round-robin can dispatch to a slot on a different
   GPU than the one the caller prepared inputs on. Detect foreign input
   GpuMats via cudaPointerGetAttributes and migrate them through a pinned
   host staging buffer (two-hop D2H+H2D; direct cross-device cudaMemcpy2D
   is unsupported and GeForce has no P2P).

3. warmUp: now pins its own CUDA device before uploading dummy inputs and
   returns success/failure; loadNetwork rejects engines whose warmup fails
   so broken slots can never join the pool silently.

4. Blackwell builder workaround: setMaxAuxStreams(0) on SM 12.x plus an
   .aux0 engine-cache suffix so pre-workaround caches rebuild.

Also: ANSFR-UnitTest gains --gpu N flag (sets CUDA_VISIBLE_DEVICES) for
single-GPU bisection, and rich diagnostics on input-copy failures.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-06 20:00:19 +10:00
co-authored by Claude Fable 5
parent 43ebd25d07
commit a4ea532f36
5 changed files with 247 additions and 11 deletions
@@ -130,13 +130,31 @@ static inline cudaError_t cudaStreamSynchronize_Safe(cudaStream_t stream) {
}
template <typename T>
void Engine<T>::warmUp(int iterations) {
bool Engine<T>::warmUp(int iterations) {
if (m_verbose) {
std::cout << "\n========================================" << std::endl;
std::cout << "Engine Warmup" << std::endl;
std::cout << "========================================" << std::endl;
}
// Ensure this thread's CUDA device matches the engine BEFORE the dummy-input
// uploads below. warmUp() does GPU work (GpuMat uploads) ahead of
// runInference()'s own device guard, so it must not rely on the caller
// having left the right device current (multi-GPU pool creation touches
// several devices on one thread).
{
int currentDev = -1;
cudaGetDevice(&currentDev);
if (currentDev != m_options.deviceIndex) {
std::cout << "Warning [Warmup]: thread was on GPU[" << currentDev
<< "], engine expects GPU[" << m_options.deviceIndex
<< "] -- switching device" << std::endl;
cudaSetDevice(m_options.deviceIndex);
}
}
bool allBatchesOk = true;
// Determine batch sizes to warm up
std::vector<int> batchSizes;
@@ -212,7 +230,9 @@ void Engine<T>::warmUp(int iterations) {
auto firstTime = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
if (!firstSuccess) {
if (m_verbose) std::cout << " ✗ ERROR: First iteration failed for batch=" << batchSize << std::endl;
std::cout << " ✗ ERROR: Warmup first iteration failed for batch=" << batchSize
<< " on GPU[" << m_options.deviceIndex << "]" << std::endl;
allBatchesOk = false;
continue;
}
@@ -231,7 +251,9 @@ void Engine<T>::warmUp(int iterations) {
for (int i = 1; i < iters; ++i) {
bool success = runInference(dummyInputs, dummyOutputs);
if (!success) {
if (m_verbose) std::cout << " ✗ ERROR: Iteration " << i << " failed" << std::endl;
std::cout << " ✗ ERROR: Warmup iteration " << i << " failed for batch="
<< batchSize << " on GPU[" << m_options.deviceIndex << "]" << std::endl;
allBatchesOk = false;
break;
}
}
@@ -262,6 +284,7 @@ void Engine<T>::warmUp(int iterations) {
std::cout << "Kernels pre-compiled for all batch sizes." << std::endl;
std::cout << "========================================\n" << std::endl;
}
return allBatchesOk;
}
template <typename T>
bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& inputs,std::vector<std::vector<std::vector<T>>>& featureVectors) {
@@ -778,6 +801,118 @@ bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& i
}
}
// Pass 1.5: INPUT DEVICE MIGRATION (multi-GPU pool dispatch).
//
// The caller prepares input GpuMats on ITS device (e.g. camera preprocessing
// on GPU 0), but round-robin dispatch may select a slot on another GPU.
// Kernels cannot touch cross-device memory without peer access: preprocessing
// a device-A GpuMat with a device-B stream raises an async illegal memory
// access that permanently poisons every CUDA context in the process
// (sticky error 700). Detect foreign inputs and migrate them onto this
// engine's device via a UVA copy (cudaMemcpyDefault routes over PCIe).
std::vector<std::vector<cv::cuda::GpuMat>> migratedInputs;
const std::vector<std::vector<cv::cuda::GpuMat>>* activeInputs = &inputs;
{
bool anyForeign = false;
for (size_t i = 0; i < numInputs && !anyForeign; ++i) {
for (const auto& img : inputs[i]) {
if (!img.data) continue;
cudaPointerAttributes attr{};
if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess &&
attr.type == cudaMemoryTypeDevice &&
attr.device != m_options.deviceIndex) {
anyForeign = true;
}
cudaGetLastError(); // clear any error the attribute query left behind
break; // first image per tensor is representative
}
}
if (anyForeign) {
migratedInputs.resize(numInputs);
for (size_t i = 0; i < numInputs; ++i) {
migratedInputs[i].reserve(inputs[i].size());
for (const auto& src : inputs[i]) {
cudaPointerAttributes attr{};
const bool foreign = src.data &&
cudaPointerGetAttributes(&attr, src.data) == cudaSuccess &&
attr.type == cudaMemoryTypeDevice &&
attr.device != m_options.deviceIndex;
cudaGetLastError();
if (!foreign) { migratedInputs[i].push_back(src); continue; }
// Two-hop copy through pinned host staging. A direct
// device-to-device cudaMemcpy2D across GPUs is not supported
// (2D copies have no peer path, and GeForce has no P2P), so:
// hop 1: strided D2H from the foreign GPU into pinned host
// hop 2: strided H2D onto this engine's GPU
// Synchronous copies keep the single staging buffer safe to
// reuse across images; ~1 ms per 1080p frame over PCIe 4.
struct PinnedStaging { // thread-local, grow-only
void* ptr = nullptr;
size_t cap = 0;
~PinnedStaging() { if (ptr) cudaFreeHost(ptr); }
};
thread_local PinnedStaging tl_staging;
const size_t rowBytes = static_cast<size_t>(src.cols) * src.elemSize();
const size_t needed = rowBytes * static_cast<size_t>(src.rows);
if (needed > tl_staging.cap) {
if (tl_staging.ptr) cudaFreeHost(tl_staging.ptr);
tl_staging.ptr = nullptr; tl_staging.cap = 0;
if (cudaMallocHost(&tl_staging.ptr, needed) != cudaSuccess) {
std::cout << "Error [MultiGPU]: pinned staging alloc failed ("
<< needed << " bytes)" << std::endl;
return false;
}
tl_staging.cap = needed;
}
// Hop 1: D2H. UVA lets us read the foreign pointer from this
// device's context; if the runtime rejects it, retry with the
// source device current.
cudaError_t mErr = cudaMemcpy2D(
tl_staging.ptr, rowBytes, src.data, src.step,
rowBytes, src.rows, cudaMemcpyDeviceToHost);
if (mErr != cudaSuccess) {
cudaGetLastError();
cudaSetDevice(attr.device);
mErr = cudaMemcpy2D(
tl_staging.ptr, rowBytes, src.data, src.step,
rowBytes, src.rows, cudaMemcpyDeviceToHost);
cudaSetDevice(m_options.deviceIndex);
}
if (mErr != cudaSuccess) {
std::cout << "Error [MultiGPU]: migration hop1 (D2H from GPU["
<< attr.device << "]) failed: " << cudaGetErrorString(mErr) << std::endl;
return false;
}
// Hop 2: H2D onto this engine's GPU.
cv::cuda::GpuMat dst(src.rows, src.cols, src.type());
mErr = cudaMemcpy2D(
dst.data, dst.step, tl_staging.ptr, rowBytes,
rowBytes, src.rows, cudaMemcpyHostToDevice);
if (mErr != cudaSuccess) {
std::cout << "Error [MultiGPU]: migration hop2 (H2D to GPU["
<< m_options.deviceIndex << "]) failed: "
<< cudaGetErrorString(mErr) << std::endl;
return false;
}
migratedInputs[i].push_back(dst);
}
}
activeInputs = &migratedInputs;
static std::atomic<bool> s_migrationLogged{false};
if (!s_migrationLogged.exchange(true)) {
std::cout << "Info [MultiGPU]: inputs resided on another GPU -- migrated to GPU["
<< m_options.deviceIndex << "] via UVA copy (logged once; recurs per"
<< " cross-device dispatch)" << std::endl;
}
}
}
// Pass 2: Preprocess + D2D copies — all on m_inferenceStream (no null stream).
//
// All OpenCV CUDA ops (convertTo, subtract, divide, split) in blobFromGpuMats
@@ -794,7 +929,7 @@ bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& i
preprocessedBuffers.reserve(numInputs);
for (size_t i = 0; i < numInputs; ++i) {
const auto& batchInput = inputs[i];
const auto& batchInput = (*activeInputs)[i];
// Preprocess on m_inferenceStream (not the null stream).
preprocessedBuffers.push_back(
@@ -811,8 +946,23 @@ bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& i
m_inferenceStream);
if (copyErr != cudaSuccess) {
// Rich diagnostics — this fires on multi-GPU slot warmup failures
// (Blackwell debugging). Identify WHICH precondition broke:
// wrong current device, empty blob (failed allocation), null dest
// buffer, or a pre-existing sticky async error.
int curDev = -1; cudaGetDevice(&curDev);
std::cout << "Error: Failed to copy input " << i
<< " to inference buffer: " << cudaGetErrorString(copyErr) << std::endl;
<< " to inference buffer: " << cudaGetErrorString(copyErr) << "\n"
<< " [diag] engine deviceIndex = " << m_options.deviceIndex
<< ", thread current device = " << curDev << "\n"
<< " [diag] src blob ptr = " << static_cast<const void*>(blobMat.data)
<< " (" << blobMat.rows << "x" << blobMat.cols
<< ", elemSize " << blobMat.elemSize() << ")\n"
<< " [diag] dst buffer ptr = " << m_buffers[i]
<< ", copySize = " << copySize << " bytes\n"
<< " [diag] inference stream = " << static_cast<const void*>(m_inferenceStream)
<< ", pending lastError = " << cudaGetErrorString(cudaGetLastError())
<< std::endl;
return false;
}
}