Fix multi-GPU inference on Blackwell (dual RTX 5080): four stacked fixes
Verified on dual RTX 5080: 355/355 detections, both GPUs in round-robin, zero CUDA errors. Requires TensorRT >= 10.16.1 in SharedCore (10.15.1 has a Myelin l2cm crash when creating a context on a second Blackwell device). 1. blobFromGpuMats: thread_local GpuMat caches (tl_blob/tl_floatImg) were device-blind — a pool thread warming GPU[1] then GPU[0] reused device-1 memory with a device-0 stream, causing an async illegal memory access that poisoned every CUDA context (sticky 700). Track owning device and rebuild the cache on device switch. 2. runInference: pool round-robin can dispatch to a slot on a different GPU than the one the caller prepared inputs on. Detect foreign input GpuMats via cudaPointerGetAttributes and migrate them through a pinned host staging buffer (two-hop D2H+H2D; direct cross-device cudaMemcpy2D is unsupported and GeForce has no P2P). 3. warmUp: now pins its own CUDA device before uploading dummy inputs and returns success/failure; loadNetwork rejects engines whose warmup fails so broken slots can never join the pool silently. 4. Blackwell builder workaround: setMaxAuxStreams(0) on SM 12.x plus an .aux0 engine-cache suffix so pre-workaround caches rebuild. Also: ANSFR-UnitTest gains --gpu N flag (sets CUDA_VISIBLE_DEVICES) for single-GPU bisection, and rich diagnostics on input-copy failures. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1435,7 +1435,15 @@ trt_cache_create_context:
|
|||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
if (m_verbose) std::cout << "\nInfo: Starting warm-up inference..." << std::endl;
|
if (m_verbose) std::cout << "\nInfo: Starting warm-up inference..." << std::endl;
|
||||||
warmUp(m_verbose ? 10 : 1);
|
if (!warmUp(m_verbose ? 10 : 1)) {
|
||||||
|
// A failed warmup means this engine cannot run inference (wrong-device
|
||||||
|
// buffers, poisoned context, ...). Report failure so pool creation
|
||||||
|
// rejects the slot instead of silently adding a broken engine that
|
||||||
|
// corrupts every subsequent inference dispatched to it.
|
||||||
|
std::cout << "Error: Warm-up inference FAILED on GPU[" << m_options.deviceIndex
|
||||||
|
<< "] -- engine rejected (loadNetwork returns false)" << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (m_verbose) std::cout << "Info: Warm-up complete" << std::endl;
|
if (m_verbose) std::cout << "Info: Warm-up complete" << std::endl;
|
||||||
|
|
||||||
} // end of trt_cache_create_context scope
|
} // end of trt_cache_create_context scope
|
||||||
@@ -1808,6 +1816,25 @@ bool Engine<T>::build(std::string onnxModelPath, const std::array<float, 3>& sub
|
|||||||
// Set hardware compatibility
|
// Set hardware compatibility
|
||||||
config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kNONE);
|
config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kNONE);
|
||||||
|
|
||||||
|
// -- Blackwell (SM 12.x) aux-stream workaround ----------------------------
|
||||||
|
// On sm_120 (RTX 50-series), TRT 10.15's Myelin runtime faults when a second
|
||||||
|
// concurrent execution context runs with auto-selected aux streams:
|
||||||
|
// "Internal Error: MyelinCheckException ... l2cm_layer_internal.cpp:174"
|
||||||
|
// followed by CUDA error 700 (illegal memory access) poisoning the context.
|
||||||
|
// Aux streams only provide intra-inference parallelism (irrelevant for the
|
||||||
|
// small per-slot models we run); throughput comes from the engine pool's
|
||||||
|
// one-context-per-slot design, which is unaffected. Force 0 aux streams on
|
||||||
|
// Blackwell until NVIDIA fixes the Myelin multi-stream path.
|
||||||
|
// NOTE: baked in at build time -- stale cached .engine files must be deleted
|
||||||
|
// for this to take effect.
|
||||||
|
#if NV_TENSORRT_MAJOR >= 10
|
||||||
|
if (prop.major >= 12) {
|
||||||
|
config->setMaxAuxStreams(0);
|
||||||
|
std::cout << "Aux streams disabled on SM " << prop.major << "." << prop.minor
|
||||||
|
<< " (Blackwell Myelin multi-stream workaround)" << std::endl;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
// -- TensorRT 10+ tactic DRAM pool ----------------------------------------
|
// -- TensorRT 10+ tactic DRAM pool ----------------------------------------
|
||||||
// Separate scratch pool for kernel-selection during build. Without this,
|
// Separate scratch pool for kernel-selection during build. Without this,
|
||||||
// tactic evaluation competes with the workspace allocation and tactics
|
// tactic evaluation competes with the workspace allocation and tactics
|
||||||
|
|||||||
@@ -130,13 +130,31 @@ static inline cudaError_t cudaStreamSynchronize_Safe(cudaStream_t stream) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
void Engine<T>::warmUp(int iterations) {
|
bool Engine<T>::warmUp(int iterations) {
|
||||||
if (m_verbose) {
|
if (m_verbose) {
|
||||||
std::cout << "\n========================================" << std::endl;
|
std::cout << "\n========================================" << std::endl;
|
||||||
std::cout << "Engine Warmup" << std::endl;
|
std::cout << "Engine Warmup" << std::endl;
|
||||||
std::cout << "========================================" << std::endl;
|
std::cout << "========================================" << std::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Ensure this thread's CUDA device matches the engine BEFORE the dummy-input
|
||||||
|
// uploads below. warmUp() does GPU work (GpuMat uploads) ahead of
|
||||||
|
// runInference()'s own device guard, so it must not rely on the caller
|
||||||
|
// having left the right device current (multi-GPU pool creation touches
|
||||||
|
// several devices on one thread).
|
||||||
|
{
|
||||||
|
int currentDev = -1;
|
||||||
|
cudaGetDevice(¤tDev);
|
||||||
|
if (currentDev != m_options.deviceIndex) {
|
||||||
|
std::cout << "Warning [Warmup]: thread was on GPU[" << currentDev
|
||||||
|
<< "], engine expects GPU[" << m_options.deviceIndex
|
||||||
|
<< "] -- switching device" << std::endl;
|
||||||
|
cudaSetDevice(m_options.deviceIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool allBatchesOk = true;
|
||||||
|
|
||||||
// Determine batch sizes to warm up
|
// Determine batch sizes to warm up
|
||||||
std::vector<int> batchSizes;
|
std::vector<int> batchSizes;
|
||||||
|
|
||||||
@@ -212,7 +230,9 @@ void Engine<T>::warmUp(int iterations) {
|
|||||||
auto firstTime = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
|
auto firstTime = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
|
||||||
|
|
||||||
if (!firstSuccess) {
|
if (!firstSuccess) {
|
||||||
if (m_verbose) std::cout << " ✗ ERROR: First iteration failed for batch=" << batchSize << std::endl;
|
std::cout << " ✗ ERROR: Warmup first iteration failed for batch=" << batchSize
|
||||||
|
<< " on GPU[" << m_options.deviceIndex << "]" << std::endl;
|
||||||
|
allBatchesOk = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,7 +251,9 @@ void Engine<T>::warmUp(int iterations) {
|
|||||||
for (int i = 1; i < iters; ++i) {
|
for (int i = 1; i < iters; ++i) {
|
||||||
bool success = runInference(dummyInputs, dummyOutputs);
|
bool success = runInference(dummyInputs, dummyOutputs);
|
||||||
if (!success) {
|
if (!success) {
|
||||||
if (m_verbose) std::cout << " ✗ ERROR: Iteration " << i << " failed" << std::endl;
|
std::cout << " ✗ ERROR: Warmup iteration " << i << " failed for batch="
|
||||||
|
<< batchSize << " on GPU[" << m_options.deviceIndex << "]" << std::endl;
|
||||||
|
allBatchesOk = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -262,6 +284,7 @@ void Engine<T>::warmUp(int iterations) {
|
|||||||
std::cout << "Kernels pre-compiled for all batch sizes." << std::endl;
|
std::cout << "Kernels pre-compiled for all batch sizes." << std::endl;
|
||||||
std::cout << "========================================\n" << std::endl;
|
std::cout << "========================================\n" << std::endl;
|
||||||
}
|
}
|
||||||
|
return allBatchesOk;
|
||||||
}
|
}
|
||||||
template <typename T>
|
template <typename T>
|
||||||
bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& inputs,std::vector<std::vector<std::vector<T>>>& featureVectors) {
|
bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& inputs,std::vector<std::vector<std::vector<T>>>& featureVectors) {
|
||||||
@@ -778,6 +801,118 @@ bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& i
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pass 1.5: INPUT DEVICE MIGRATION (multi-GPU pool dispatch).
|
||||||
|
//
|
||||||
|
// The caller prepares input GpuMats on ITS device (e.g. camera preprocessing
|
||||||
|
// on GPU 0), but round-robin dispatch may select a slot on another GPU.
|
||||||
|
// Kernels cannot touch cross-device memory without peer access: preprocessing
|
||||||
|
// a device-A GpuMat with a device-B stream raises an async illegal memory
|
||||||
|
// access that permanently poisons every CUDA context in the process
|
||||||
|
// (sticky error 700). Detect foreign inputs and migrate them onto this
|
||||||
|
// engine's device via a UVA copy (cudaMemcpyDefault routes over PCIe).
|
||||||
|
std::vector<std::vector<cv::cuda::GpuMat>> migratedInputs;
|
||||||
|
const std::vector<std::vector<cv::cuda::GpuMat>>* activeInputs = &inputs;
|
||||||
|
{
|
||||||
|
bool anyForeign = false;
|
||||||
|
for (size_t i = 0; i < numInputs && !anyForeign; ++i) {
|
||||||
|
for (const auto& img : inputs[i]) {
|
||||||
|
if (!img.data) continue;
|
||||||
|
cudaPointerAttributes attr{};
|
||||||
|
if (cudaPointerGetAttributes(&attr, img.data) == cudaSuccess &&
|
||||||
|
attr.type == cudaMemoryTypeDevice &&
|
||||||
|
attr.device != m_options.deviceIndex) {
|
||||||
|
anyForeign = true;
|
||||||
|
}
|
||||||
|
cudaGetLastError(); // clear any error the attribute query left behind
|
||||||
|
break; // first image per tensor is representative
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (anyForeign) {
|
||||||
|
migratedInputs.resize(numInputs);
|
||||||
|
for (size_t i = 0; i < numInputs; ++i) {
|
||||||
|
migratedInputs[i].reserve(inputs[i].size());
|
||||||
|
for (const auto& src : inputs[i]) {
|
||||||
|
cudaPointerAttributes attr{};
|
||||||
|
const bool foreign = src.data &&
|
||||||
|
cudaPointerGetAttributes(&attr, src.data) == cudaSuccess &&
|
||||||
|
attr.type == cudaMemoryTypeDevice &&
|
||||||
|
attr.device != m_options.deviceIndex;
|
||||||
|
cudaGetLastError();
|
||||||
|
if (!foreign) { migratedInputs[i].push_back(src); continue; }
|
||||||
|
|
||||||
|
// Two-hop copy through pinned host staging. A direct
|
||||||
|
// device-to-device cudaMemcpy2D across GPUs is not supported
|
||||||
|
// (2D copies have no peer path, and GeForce has no P2P), so:
|
||||||
|
// hop 1: strided D2H from the foreign GPU into pinned host
|
||||||
|
// hop 2: strided H2D onto this engine's GPU
|
||||||
|
// Synchronous copies keep the single staging buffer safe to
|
||||||
|
// reuse across images; ~1 ms per 1080p frame over PCIe 4.
|
||||||
|
struct PinnedStaging { // thread-local, grow-only
|
||||||
|
void* ptr = nullptr;
|
||||||
|
size_t cap = 0;
|
||||||
|
~PinnedStaging() { if (ptr) cudaFreeHost(ptr); }
|
||||||
|
};
|
||||||
|
thread_local PinnedStaging tl_staging;
|
||||||
|
|
||||||
|
const size_t rowBytes = static_cast<size_t>(src.cols) * src.elemSize();
|
||||||
|
const size_t needed = rowBytes * static_cast<size_t>(src.rows);
|
||||||
|
if (needed > tl_staging.cap) {
|
||||||
|
if (tl_staging.ptr) cudaFreeHost(tl_staging.ptr);
|
||||||
|
tl_staging.ptr = nullptr; tl_staging.cap = 0;
|
||||||
|
if (cudaMallocHost(&tl_staging.ptr, needed) != cudaSuccess) {
|
||||||
|
std::cout << "Error [MultiGPU]: pinned staging alloc failed ("
|
||||||
|
<< needed << " bytes)" << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
tl_staging.cap = needed;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hop 1: D2H. UVA lets us read the foreign pointer from this
|
||||||
|
// device's context; if the runtime rejects it, retry with the
|
||||||
|
// source device current.
|
||||||
|
cudaError_t mErr = cudaMemcpy2D(
|
||||||
|
tl_staging.ptr, rowBytes, src.data, src.step,
|
||||||
|
rowBytes, src.rows, cudaMemcpyDeviceToHost);
|
||||||
|
if (mErr != cudaSuccess) {
|
||||||
|
cudaGetLastError();
|
||||||
|
cudaSetDevice(attr.device);
|
||||||
|
mErr = cudaMemcpy2D(
|
||||||
|
tl_staging.ptr, rowBytes, src.data, src.step,
|
||||||
|
rowBytes, src.rows, cudaMemcpyDeviceToHost);
|
||||||
|
cudaSetDevice(m_options.deviceIndex);
|
||||||
|
}
|
||||||
|
if (mErr != cudaSuccess) {
|
||||||
|
std::cout << "Error [MultiGPU]: migration hop1 (D2H from GPU["
|
||||||
|
<< attr.device << "]) failed: " << cudaGetErrorString(mErr) << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hop 2: H2D onto this engine's GPU.
|
||||||
|
cv::cuda::GpuMat dst(src.rows, src.cols, src.type());
|
||||||
|
mErr = cudaMemcpy2D(
|
||||||
|
dst.data, dst.step, tl_staging.ptr, rowBytes,
|
||||||
|
rowBytes, src.rows, cudaMemcpyHostToDevice);
|
||||||
|
if (mErr != cudaSuccess) {
|
||||||
|
std::cout << "Error [MultiGPU]: migration hop2 (H2D to GPU["
|
||||||
|
<< m_options.deviceIndex << "]) failed: "
|
||||||
|
<< cudaGetErrorString(mErr) << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
migratedInputs[i].push_back(dst);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
activeInputs = &migratedInputs;
|
||||||
|
|
||||||
|
static std::atomic<bool> s_migrationLogged{false};
|
||||||
|
if (!s_migrationLogged.exchange(true)) {
|
||||||
|
std::cout << "Info [MultiGPU]: inputs resided on another GPU -- migrated to GPU["
|
||||||
|
<< m_options.deviceIndex << "] via UVA copy (logged once; recurs per"
|
||||||
|
<< " cross-device dispatch)" << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Pass 2: Preprocess + D2D copies — all on m_inferenceStream (no null stream).
|
// Pass 2: Preprocess + D2D copies — all on m_inferenceStream (no null stream).
|
||||||
//
|
//
|
||||||
// All OpenCV CUDA ops (convertTo, subtract, divide, split) in blobFromGpuMats
|
// All OpenCV CUDA ops (convertTo, subtract, divide, split) in blobFromGpuMats
|
||||||
@@ -794,7 +929,7 @@ bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& i
|
|||||||
preprocessedBuffers.reserve(numInputs);
|
preprocessedBuffers.reserve(numInputs);
|
||||||
|
|
||||||
for (size_t i = 0; i < numInputs; ++i) {
|
for (size_t i = 0; i < numInputs; ++i) {
|
||||||
const auto& batchInput = inputs[i];
|
const auto& batchInput = (*activeInputs)[i];
|
||||||
|
|
||||||
// Preprocess on m_inferenceStream (not the null stream).
|
// Preprocess on m_inferenceStream (not the null stream).
|
||||||
preprocessedBuffers.push_back(
|
preprocessedBuffers.push_back(
|
||||||
@@ -811,8 +946,23 @@ bool Engine<T>::runInference(const std::vector<std::vector<cv::cuda::GpuMat>>& i
|
|||||||
m_inferenceStream);
|
m_inferenceStream);
|
||||||
|
|
||||||
if (copyErr != cudaSuccess) {
|
if (copyErr != cudaSuccess) {
|
||||||
|
// Rich diagnostics — this fires on multi-GPU slot warmup failures
|
||||||
|
// (Blackwell debugging). Identify WHICH precondition broke:
|
||||||
|
// wrong current device, empty blob (failed allocation), null dest
|
||||||
|
// buffer, or a pre-existing sticky async error.
|
||||||
|
int curDev = -1; cudaGetDevice(&curDev);
|
||||||
std::cout << "Error: Failed to copy input " << i
|
std::cout << "Error: Failed to copy input " << i
|
||||||
<< " to inference buffer: " << cudaGetErrorString(copyErr) << std::endl;
|
<< " to inference buffer: " << cudaGetErrorString(copyErr) << "\n"
|
||||||
|
<< " [diag] engine deviceIndex = " << m_options.deviceIndex
|
||||||
|
<< ", thread current device = " << curDev << "\n"
|
||||||
|
<< " [diag] src blob ptr = " << static_cast<const void*>(blobMat.data)
|
||||||
|
<< " (" << blobMat.rows << "x" << blobMat.cols
|
||||||
|
<< ", elemSize " << blobMat.elemSize() << ")\n"
|
||||||
|
<< " [diag] dst buffer ptr = " << m_buffers[i]
|
||||||
|
<< ", copySize = " << copySize << " bytes\n"
|
||||||
|
<< " [diag] inference stream = " << static_cast<const void*>(m_inferenceStream)
|
||||||
|
<< ", pending lastError = " << cudaGetErrorString(cudaGetLastError())
|
||||||
|
<< std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -207,6 +207,21 @@ std::string Engine<T>::serializeEngineOptions(const ANSCENTER::Options& options,
|
|||||||
+ std::to_string(CUDNN_MAJOR) + "."
|
+ std::to_string(CUDNN_MAJOR) + "."
|
||||||
+ std::to_string(CUDNN_MINOR);
|
+ std::to_string(CUDNN_MINOR);
|
||||||
|
|
||||||
|
// -- Blackwell aux-stream workaround marker --------------------------------
|
||||||
|
// On SM 12.x the builder forces maxAuxStreams=0 (see EngineBuildLoadNetwork.inl:
|
||||||
|
// TRT 10.15 Myelin faults with concurrent contexts + aux streams). Tag those
|
||||||
|
// engines so caches built *before* the workaround (with aux streams baked in)
|
||||||
|
// miss and rebuild automatically instead of loading and crashing.
|
||||||
|
#if NV_TENSORRT_MAJOR >= 10
|
||||||
|
{
|
||||||
|
cudaDeviceProp prop{};
|
||||||
|
if (cudaGetDeviceProperties(&prop, options.deviceIndex) == cudaSuccess
|
||||||
|
&& prop.major >= 12) {
|
||||||
|
engineName += ".aux0";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
return engineName;
|
return engineName;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,6 +246,32 @@ cv::cuda::GpuMat Engine<T>::blobFromGpuMats(const std::vector<cv::cuda::GpuMat>
|
|||||||
thread_local cv::cuda::GpuMat tl_blob;
|
thread_local cv::cuda::GpuMat tl_blob;
|
||||||
thread_local cv::cuda::GpuMat tl_floatImg;
|
thread_local cv::cuda::GpuMat tl_floatImg;
|
||||||
thread_local int tl_blobMaxElems = 0;
|
thread_local int tl_blobMaxElems = 0;
|
||||||
|
thread_local int tl_blobDevice = -1;
|
||||||
|
|
||||||
|
// DEVICE AFFINITY: the cached GpuMats live on ONE device. When the same
|
||||||
|
// thread serves engines on different GPUs (multi-GPU pool creation warms
|
||||||
|
// probe on GPU A, then slots on GPU B), reusing a device-A blob with a
|
||||||
|
// device-B stream causes invalid-argument launches and an async illegal
|
||||||
|
// memory access that permanently poisons both CUDA contexts (sticky 700).
|
||||||
|
// Detect the device switch and rebuild the cache on the current device.
|
||||||
|
{
|
||||||
|
int curDev = -1;
|
||||||
|
cudaGetDevice(&curDev);
|
||||||
|
if (curDev != tl_blobDevice) {
|
||||||
|
if (tl_blobDevice >= 0) {
|
||||||
|
// Free on the owning device — the async allocator frees via the
|
||||||
|
// CURRENT device's null-stream pool, so switch, release, switch back.
|
||||||
|
cudaSetDevice(tl_blobDevice);
|
||||||
|
tl_blob.release();
|
||||||
|
tl_floatImg.release();
|
||||||
|
cudaSetDevice(curDev);
|
||||||
|
ANS_DBG("TRT_Preproc", "blobFromGpuMats: device switch GPU[%d]->GPU[%d], preproc cache rebuilt",
|
||||||
|
tl_blobDevice, curDev);
|
||||||
|
}
|
||||||
|
tl_blobMaxElems = 0;
|
||||||
|
tl_blobDevice = curDev;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (totalElems > tl_blobMaxElems) {
|
if (totalElems > tl_blobMaxElems) {
|
||||||
tl_blob = cv::cuda::GpuMat(1, totalElems, CV_32FC1);
|
tl_blob = cv::cuda::GpuMat(1, totalElems, CV_32FC1);
|
||||||
|
|||||||
@@ -240,7 +240,9 @@ public:
|
|||||||
void clearGpuBuffers();
|
void clearGpuBuffers();
|
||||||
int getBindingIndexByName(const std::string& name);
|
int getBindingIndexByName(const std::string& name);
|
||||||
void UpdateOptions(const ANSCENTER::Options& options) { m_options = options; }
|
void UpdateOptions(const ANSCENTER::Options& options) { m_options = options; }
|
||||||
void warmUp(int iterations = 5);
|
// Returns false if any warmup inference failed — callers must treat the
|
||||||
|
// engine as unusable (a slot that fails warmup must NOT join the pool).
|
||||||
|
bool warmUp(int iterations = 5);
|
||||||
void setVerbose(bool v) { m_verbose = v; }
|
void setVerbose(bool v) { m_verbose = v; }
|
||||||
void setDisableGraphs(bool v) { m_disableGraphs = v; }
|
void setDisableGraphs(bool v) { m_disableGraphs = v; }
|
||||||
|
|
||||||
|
|||||||
@@ -731,7 +731,7 @@ int ANSVISTest() {
|
|||||||
|
|
||||||
//std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\ANSVIS_Issues\\face.mp4";
|
//std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\ANSVIS_Issues\\face.mp4";
|
||||||
//std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\TestFR\\Face31.mp4";
|
//std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\TestFR\\Face31.mp4";
|
||||||
std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\classroom.mp4";
|
std::string videoFilePath = "C:\\Programs\\DemoAssets\\Videos\\classroom.mp4";
|
||||||
//std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\TestFR\\school1.mp4";// "C:\\ProgramData\\ANSCENTER\\Shared\\classroom.mp4";
|
//std::string videoFilePath = "E:\\Programs\\DemoAssets\\Videos\\TestFR\\school1.mp4";// "C:\\ProgramData\\ANSCENTER\\Shared\\classroom.mp4";
|
||||||
|
|
||||||
const char* configFilePath = "";
|
const char* configFilePath = "";
|
||||||
@@ -1713,15 +1713,31 @@ int SCRFDVideoRepro()
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main()
|
int main(int argc, char** argv)
|
||||||
{
|
{
|
||||||
|
// ── GPU visibility bisect (Blackwell Myelin l2cm debugging) ─────────────
|
||||||
|
// Usage: ANSFR-UnitTest.exe --gpu 0 → only GPU0 visible (Test A, display GPU)
|
||||||
|
// ANSFR-UnitTest.exe --gpu 1 → only GPU1 visible (Test B, headless GPU)
|
||||||
|
// ANSFR-UnitTest.exe → all GPUs (default behaviour)
|
||||||
|
// Sets CUDA_VISIBLE_DEVICES before the first CUDA call in this process, so
|
||||||
|
// the CUDA runtime only enumerates the chosen physical GPU (it becomes
|
||||||
|
// device 0 inside the process).
|
||||||
|
for (int i = 1; i + 1 < argc; ++i) {
|
||||||
|
if (std::string(argv[i]) == "--gpu") {
|
||||||
|
_putenv_s("CUDA_VISIBLE_DEVICES", argv[i + 1]);
|
||||||
|
std::cout << "==============================================\n"
|
||||||
|
<< " GPU BISECT: CUDA_VISIBLE_DEVICES=" << argv[i + 1] << "\n"
|
||||||
|
<< " (only this physical GPU is visible to CUDA)\n"
|
||||||
|
<< "==============================================" << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
//FaceDetectorTest();
|
//FaceDetectorTest();
|
||||||
//TestFaceRecognition();
|
//TestFaceRecognition();
|
||||||
//TestCompleteFR1();
|
//TestCompleteFR1();
|
||||||
|
|
||||||
//ANSVISImageTest();
|
//ANSVISImageTest();
|
||||||
//ANSVISTest();
|
ANSVISTest();
|
||||||
//ANSVISTestFilePlayer();
|
//ANSVISTestFilePlayer();
|
||||||
// ANSVISRecordingTest();
|
// ANSVISRecordingTest();
|
||||||
//FRStressTest();
|
//FRStressTest();
|
||||||
@@ -1735,7 +1751,7 @@ int main()
|
|||||||
// TestCompleteFR();
|
// TestCompleteFR();
|
||||||
//TestFaceRecognition();
|
//TestFaceRecognition();
|
||||||
//FaceRecognitionBenchmark();
|
//FaceRecognitionBenchmark();
|
||||||
SCRFDVideoRepro();
|
//SCRFDVideoRepro();
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user