From a4ea532f3627bc6a4fd5dca19ceeb4cdb07c48d9 Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Mon, 6 Jul 2026 20:00:19 +1000 Subject: [PATCH] Fix multi-GPU inference on Blackwell (dual RTX 5080): four stacked fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified on dual RTX 5080: 355/355 detections, both GPUs in round-robin, zero CUDA errors. Requires TensorRT >= 10.16.1 in SharedCore (10.15.1 has a Myelin l2cm crash when creating a context on a second Blackwell device). 1. blobFromGpuMats: thread_local GpuMat caches (tl_blob/tl_floatImg) were device-blind — a pool thread warming GPU[1] then GPU[0] reused device-1 memory with a device-0 stream, causing an async illegal memory access that poisoned every CUDA context (sticky 700). Track owning device and rebuild the cache on device switch. 2. runInference: pool round-robin can dispatch to a slot on a different GPU than the one the caller prepared inputs on. Detect foreign input GpuMats via cudaPointerGetAttributes and migrate them through a pinned host staging buffer (two-hop D2H+H2D; direct cross-device cudaMemcpy2D is unsupported and GeForce has no P2P). 3. warmUp: now pins its own CUDA device before uploading dummy inputs and returns success/failure; loadNetwork rejects engines whose warmup fails so broken slots can never join the pool silently. 4. Blackwell builder workaround: setMaxAuxStreams(0) on SM 12.x plus an .aux0 engine-cache suffix so pre-workaround caches rebuild. Also: ANSFR-UnitTest gains --gpu N flag (sets CUDA_VISIBLE_DEVICES) for single-GPU bisection, and rich diagnostics on input-copy failures. Co-Authored-By: Claude Fable 5 --- .../include/engine/EngineBuildLoadNetwork.inl | 29 +++- .../include/engine/EngineRunInference.inl | 160 +++++++++++++++++- .../include/engine/EngineUtilities.inl | 41 +++++ modules/ANSODEngine/engine.h | 4 +- tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp | 24 ++- 5 files changed, 247 insertions(+), 11 deletions(-) diff --git a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl index 33fafde..16ac459 100644 --- a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl +++ b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl @@ -1435,7 +1435,15 @@ trt_cache_create_context: // ============================================================================ if (m_verbose) std::cout << "\nInfo: Starting warm-up inference..." << std::endl; - warmUp(m_verbose ? 10 : 1); + if (!warmUp(m_verbose ? 10 : 1)) { + // A failed warmup means this engine cannot run inference (wrong-device + // buffers, poisoned context, ...). Report failure so pool creation + // rejects the slot instead of silently adding a broken engine that + // corrupts every subsequent inference dispatched to it. + std::cout << "Error: Warm-up inference FAILED on GPU[" << m_options.deviceIndex + << "] -- engine rejected (loadNetwork returns false)" << std::endl; + return false; + } if (m_verbose) std::cout << "Info: Warm-up complete" << std::endl; } // end of trt_cache_create_context scope @@ -1808,6 +1816,25 @@ bool Engine::build(std::string onnxModelPath, const std::array& sub // Set hardware compatibility config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kNONE); + // -- Blackwell (SM 12.x) aux-stream workaround ---------------------------- + // On sm_120 (RTX 50-series), TRT 10.15's Myelin runtime faults when a second + // concurrent execution context runs with auto-selected aux streams: + // "Internal Error: MyelinCheckException ... l2cm_layer_internal.cpp:174" + // followed by CUDA error 700 (illegal memory access) poisoning the context. + // Aux streams only provide intra-inference parallelism (irrelevant for the + // small per-slot models we run); throughput comes from the engine pool's + // one-context-per-slot design, which is unaffected. Force 0 aux streams on + // Blackwell until NVIDIA fixes the Myelin multi-stream path. + // NOTE: baked in at build time -- stale cached .engine files must be deleted + // for this to take effect. +#if NV_TENSORRT_MAJOR >= 10 + if (prop.major >= 12) { + config->setMaxAuxStreams(0); + std::cout << "Aux streams disabled on SM " << prop.major << "." << prop.minor + << " (Blackwell Myelin multi-stream workaround)" << std::endl; + } +#endif + // -- TensorRT 10+ tactic DRAM pool ---------------------------------------- // Separate scratch pool for kernel-selection during build. Without this, // tactic evaluation competes with the workspace allocation and tactics diff --git a/engines/TensorRTAPI/include/engine/EngineRunInference.inl b/engines/TensorRTAPI/include/engine/EngineRunInference.inl index 1944486..5878d0d 100644 --- a/engines/TensorRTAPI/include/engine/EngineRunInference.inl +++ b/engines/TensorRTAPI/include/engine/EngineRunInference.inl @@ -130,13 +130,31 @@ static inline cudaError_t cudaStreamSynchronize_Safe(cudaStream_t stream) { } template -void Engine::warmUp(int iterations) { +bool Engine::warmUp(int iterations) { if (m_verbose) { std::cout << "\n========================================" << std::endl; std::cout << "Engine Warmup" << std::endl; std::cout << "========================================" << std::endl; } + // Ensure this thread's CUDA device matches the engine BEFORE the dummy-input + // uploads below. warmUp() does GPU work (GpuMat uploads) ahead of + // runInference()'s own device guard, so it must not rely on the caller + // having left the right device current (multi-GPU pool creation touches + // several devices on one thread). + { + int currentDev = -1; + cudaGetDevice(¤tDev); + if (currentDev != m_options.deviceIndex) { + std::cout << "Warning [Warmup]: thread was on GPU[" << currentDev + << "], engine expects GPU[" << m_options.deviceIndex + << "] -- switching device" << std::endl; + cudaSetDevice(m_options.deviceIndex); + } + } + + bool allBatchesOk = true; + // Determine batch sizes to warm up std::vector batchSizes; @@ -212,7 +230,9 @@ void Engine::warmUp(int iterations) { auto firstTime = std::chrono::duration_cast(end - start).count(); if (!firstSuccess) { - if (m_verbose) std::cout << " ✗ ERROR: First iteration failed for batch=" << batchSize << std::endl; + std::cout << " ✗ ERROR: Warmup first iteration failed for batch=" << batchSize + << " on GPU[" << m_options.deviceIndex << "]" << std::endl; + allBatchesOk = false; continue; } @@ -231,7 +251,9 @@ void Engine::warmUp(int iterations) { for (int i = 1; i < iters; ++i) { bool success = runInference(dummyInputs, dummyOutputs); if (!success) { - if (m_verbose) std::cout << " ✗ ERROR: Iteration " << i << " failed" << std::endl; + std::cout << " ✗ ERROR: Warmup iteration " << i << " failed for batch=" + << batchSize << " on GPU[" << m_options.deviceIndex << "]" << std::endl; + allBatchesOk = false; break; } } @@ -262,6 +284,7 @@ void Engine::warmUp(int iterations) { std::cout << "Kernels pre-compiled for all batch sizes." << std::endl; std::cout << "========================================\n" << std::endl; } + return allBatchesOk; } template bool Engine::runInference(const std::vector>& inputs,std::vector>>& featureVectors) { @@ -778,6 +801,118 @@ bool Engine::runInference(const std::vector>& i } } + // Pass 1.5: INPUT DEVICE MIGRATION (multi-GPU pool dispatch). + // + // The caller prepares input GpuMats on ITS device (e.g. camera preprocessing + // on GPU 0), but round-robin dispatch may select a slot on another GPU. + // Kernels cannot touch cross-device memory without peer access: preprocessing + // a device-A GpuMat with a device-B stream raises an async illegal memory + // access that permanently poisons every CUDA context in the process + // (sticky error 700). Detect foreign inputs and migrate them onto this + // engine's device via a UVA copy (cudaMemcpyDefault routes over PCIe). + std::vector> migratedInputs; + const std::vector>* activeInputs = &inputs; + { + bool anyForeign = false; + for (size_t i = 0; i < numInputs && !anyForeign; ++i) { + for (const auto& img : inputs[i]) { + if (!img.data) continue; + cudaPointerAttributes attr{}; + if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess && + attr.type == cudaMemoryTypeDevice && + attr.device != m_options.deviceIndex) { + anyForeign = true; + } + cudaGetLastError(); // clear any error the attribute query left behind + break; // first image per tensor is representative + } + } + + if (anyForeign) { + migratedInputs.resize(numInputs); + for (size_t i = 0; i < numInputs; ++i) { + migratedInputs[i].reserve(inputs[i].size()); + for (const auto& src : inputs[i]) { + cudaPointerAttributes attr{}; + const bool foreign = src.data && + cudaPointerGetAttributes(&attr, src.data) == cudaSuccess && + attr.type == cudaMemoryTypeDevice && + attr.device != m_options.deviceIndex; + cudaGetLastError(); + if (!foreign) { migratedInputs[i].push_back(src); continue; } + + // Two-hop copy through pinned host staging. A direct + // device-to-device cudaMemcpy2D across GPUs is not supported + // (2D copies have no peer path, and GeForce has no P2P), so: + // hop 1: strided D2H from the foreign GPU into pinned host + // hop 2: strided H2D onto this engine's GPU + // Synchronous copies keep the single staging buffer safe to + // reuse across images; ~1 ms per 1080p frame over PCIe 4. + struct PinnedStaging { // thread-local, grow-only + void* ptr = nullptr; + size_t cap = 0; + ~PinnedStaging() { if (ptr) cudaFreeHost(ptr); } + }; + thread_local PinnedStaging tl_staging; + + const size_t rowBytes = static_cast(src.cols) * src.elemSize(); + const size_t needed = rowBytes * static_cast(src.rows); + if (needed > tl_staging.cap) { + if (tl_staging.ptr) cudaFreeHost(tl_staging.ptr); + tl_staging.ptr = nullptr; tl_staging.cap = 0; + if (cudaMallocHost(&tl_staging.ptr, needed) != cudaSuccess) { + std::cout << "Error [MultiGPU]: pinned staging alloc failed (" + << needed << " bytes)" << std::endl; + return false; + } + tl_staging.cap = needed; + } + + // Hop 1: D2H. UVA lets us read the foreign pointer from this + // device's context; if the runtime rejects it, retry with the + // source device current. + cudaError_t mErr = cudaMemcpy2D( + tl_staging.ptr, rowBytes, src.data, src.step, + rowBytes, src.rows, cudaMemcpyDeviceToHost); + if (mErr != cudaSuccess) { + cudaGetLastError(); + cudaSetDevice(attr.device); + mErr = cudaMemcpy2D( + tl_staging.ptr, rowBytes, src.data, src.step, + rowBytes, src.rows, cudaMemcpyDeviceToHost); + cudaSetDevice(m_options.deviceIndex); + } + if (mErr != cudaSuccess) { + std::cout << "Error [MultiGPU]: migration hop1 (D2H from GPU[" + << attr.device << "]) failed: " << cudaGetErrorString(mErr) << std::endl; + return false; + } + + // Hop 2: H2D onto this engine's GPU. + cv::cuda::GpuMat dst(src.rows, src.cols, src.type()); + mErr = cudaMemcpy2D( + dst.data, dst.step, tl_staging.ptr, rowBytes, + rowBytes, src.rows, cudaMemcpyHostToDevice); + if (mErr != cudaSuccess) { + std::cout << "Error [MultiGPU]: migration hop2 (H2D to GPU[" + << m_options.deviceIndex << "]) failed: " + << cudaGetErrorString(mErr) << std::endl; + return false; + } + migratedInputs[i].push_back(dst); + } + } + activeInputs = &migratedInputs; + + static std::atomic s_migrationLogged{false}; + if (!s_migrationLogged.exchange(true)) { + std::cout << "Info [MultiGPU]: inputs resided on another GPU -- migrated to GPU[" + << m_options.deviceIndex << "] via UVA copy (logged once; recurs per" + << " cross-device dispatch)" << std::endl; + } + } + } + // Pass 2: Preprocess + D2D copies — all on m_inferenceStream (no null stream). // // All OpenCV CUDA ops (convertTo, subtract, divide, split) in blobFromGpuMats @@ -794,7 +929,7 @@ bool Engine::runInference(const std::vector>& i preprocessedBuffers.reserve(numInputs); for (size_t i = 0; i < numInputs; ++i) { - const auto& batchInput = inputs[i]; + const auto& batchInput = (*activeInputs)[i]; // Preprocess on m_inferenceStream (not the null stream). preprocessedBuffers.push_back( @@ -811,8 +946,23 @@ bool Engine::runInference(const std::vector>& i m_inferenceStream); if (copyErr != cudaSuccess) { + // Rich diagnostics — this fires on multi-GPU slot warmup failures + // (Blackwell debugging). Identify WHICH precondition broke: + // wrong current device, empty blob (failed allocation), null dest + // buffer, or a pre-existing sticky async error. + int curDev = -1; cudaGetDevice(&curDev); std::cout << "Error: Failed to copy input " << i - << " to inference buffer: " << cudaGetErrorString(copyErr) << std::endl; + << " to inference buffer: " << cudaGetErrorString(copyErr) << "\n" + << " [diag] engine deviceIndex = " << m_options.deviceIndex + << ", thread current device = " << curDev << "\n" + << " [diag] src blob ptr = " << static_cast(blobMat.data) + << " (" << blobMat.rows << "x" << blobMat.cols + << ", elemSize " << blobMat.elemSize() << ")\n" + << " [diag] dst buffer ptr = " << m_buffers[i] + << ", copySize = " << copySize << " bytes\n" + << " [diag] inference stream = " << static_cast(m_inferenceStream) + << ", pending lastError = " << cudaGetErrorString(cudaGetLastError()) + << std::endl; return false; } } diff --git a/engines/TensorRTAPI/include/engine/EngineUtilities.inl b/engines/TensorRTAPI/include/engine/EngineUtilities.inl index d366775..15b6f6d 100644 --- a/engines/TensorRTAPI/include/engine/EngineUtilities.inl +++ b/engines/TensorRTAPI/include/engine/EngineUtilities.inl @@ -207,6 +207,21 @@ std::string Engine::serializeEngineOptions(const ANSCENTER::Options& options, + std::to_string(CUDNN_MAJOR) + "." + std::to_string(CUDNN_MINOR); + // -- Blackwell aux-stream workaround marker -------------------------------- + // On SM 12.x the builder forces maxAuxStreams=0 (see EngineBuildLoadNetwork.inl: + // TRT 10.15 Myelin faults with concurrent contexts + aux streams). Tag those + // engines so caches built *before* the workaround (with aux streams baked in) + // miss and rebuild automatically instead of loading and crashing. +#if NV_TENSORRT_MAJOR >= 10 + { + cudaDeviceProp prop{}; + if (cudaGetDeviceProperties(&prop, options.deviceIndex) == cudaSuccess + && prop.major >= 12) { + engineName += ".aux0"; + } + } +#endif + return engineName; } @@ -231,6 +246,32 @@ cv::cuda::GpuMat Engine::blobFromGpuMats(const std::vector thread_local cv::cuda::GpuMat tl_blob; thread_local cv::cuda::GpuMat tl_floatImg; thread_local int tl_blobMaxElems = 0; + thread_local int tl_blobDevice = -1; + + // DEVICE AFFINITY: the cached GpuMats live on ONE device. When the same + // thread serves engines on different GPUs (multi-GPU pool creation warms + // probe on GPU A, then slots on GPU B), reusing a device-A blob with a + // device-B stream causes invalid-argument launches and an async illegal + // memory access that permanently poisons both CUDA contexts (sticky 700). + // Detect the device switch and rebuild the cache on the current device. + { + int curDev = -1; + cudaGetDevice(&curDev); + if (curDev != tl_blobDevice) { + if (tl_blobDevice >= 0) { + // Free on the owning device — the async allocator frees via the + // CURRENT device's null-stream pool, so switch, release, switch back. + cudaSetDevice(tl_blobDevice); + tl_blob.release(); + tl_floatImg.release(); + cudaSetDevice(curDev); + ANS_DBG("TRT_Preproc", "blobFromGpuMats: device switch GPU[%d]->GPU[%d], preproc cache rebuilt", + tl_blobDevice, curDev); + } + tl_blobMaxElems = 0; + tl_blobDevice = curDev; + } + } if (totalElems > tl_blobMaxElems) { tl_blob = cv::cuda::GpuMat(1, totalElems, CV_32FC1); diff --git a/modules/ANSODEngine/engine.h b/modules/ANSODEngine/engine.h index d00228e..b0966b2 100644 --- a/modules/ANSODEngine/engine.h +++ b/modules/ANSODEngine/engine.h @@ -240,7 +240,9 @@ public: void clearGpuBuffers(); int getBindingIndexByName(const std::string& name); void UpdateOptions(const ANSCENTER::Options& options) { m_options = options; } - void warmUp(int iterations = 5); + // Returns false if any warmup inference failed — callers must treat the + // engine as unusable (a slot that fails warmup must NOT join the pool). + bool warmUp(int iterations = 5); void setVerbose(bool v) { m_verbose = v; } void setDisableGraphs(bool v) { m_disableGraphs = v; } diff --git a/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp b/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp index 34b1ba8..a7cd09c 100644 --- a/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp +++ b/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp @@ -731,7 +731,7 @@ int ANSVISTest() { //std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\ANSVIS_Issues\\face.mp4"; //std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\TestFR\\Face31.mp4"; - std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\classroom.mp4"; + std::string videoFilePath = "C:\\Programs\\DemoAssets\\Videos\\classroom.mp4"; //std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\TestFR\\school1.mp4";// "C:\\ProgramData\\ANSCENTER\\Shared\\classroom.mp4"; const char* configFilePath = ""; @@ -1713,15 +1713,31 @@ int SCRFDVideoRepro() return 0; } -int main() +int main(int argc, char** argv) { + // ── GPU visibility bisect (Blackwell Myelin l2cm debugging) ───────────── + // Usage: ANSFR-UnitTest.exe --gpu 0 → only GPU0 visible (Test A, display GPU) + // ANSFR-UnitTest.exe --gpu 1 → only GPU1 visible (Test B, headless GPU) + // ANSFR-UnitTest.exe → all GPUs (default behaviour) + // Sets CUDA_VISIBLE_DEVICES before the first CUDA call in this process, so + // the CUDA runtime only enumerates the chosen physical GPU (it becomes + // device 0 inside the process). + for (int i = 1; i + 1 < argc; ++i) { + if (std::string(argv[i]) == "--gpu") { + _putenv_s("CUDA_VISIBLE_DEVICES", argv[i + 1]); + std::cout << "==============================================\n" + << " GPU BISECT: CUDA_VISIBLE_DEVICES=" << argv[i + 1] << "\n" + << " (only this physical GPU is visible to CUDA)\n" + << "==============================================" << std::endl; + } + } //FaceDetectorTest(); //TestFaceRecognition(); //TestCompleteFR1(); //ANSVISImageTest(); - //ANSVISTest(); + ANSVISTest(); //ANSVISTestFilePlayer(); // ANSVISRecordingTest(); //FRStressTest(); @@ -1735,7 +1751,7 @@ int main() // TestCompleteFR(); //TestFaceRecognition(); //FaceRecognitionBenchmark(); - SCRFDVideoRepro(); + //SCRFDVideoRepro(); return 0; }