From cff2a90fad0d186378bb2f7bb785187e4aa5f838 Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Sat, 2 May 2026 12:24:53 +1000 Subject: [PATCH] Support multiple GPUs --- .../ANSTrainingEngine/ANSTrainingEngine.cpp | 99 +++++++++++++++++++ modules/ANSTrainingEngine/ANSTrainingEngine.h | 8 ++ .../ANSYOLOCLTrainingEngine.cpp | 8 +- .../ANSYOLOOBBTrainingEngine.cpp | 8 +- .../ANSYOLOSEGTrainingEngine.cpp | 8 +- .../ANSYOLOTrainingEngine.cpp | 10 +- tests/ANSODTRE-UnitTest/ANSODTRE-UnitTest.cpp | 22 ++--- 7 files changed, 135 insertions(+), 28 deletions(-) diff --git a/modules/ANSTrainingEngine/ANSTrainingEngine.cpp b/modules/ANSTrainingEngine/ANSTrainingEngine.cpp index c1b6d2c..049b5dc 100644 --- a/modules/ANSTrainingEngine/ANSTrainingEngine.cpp +++ b/modules/ANSTrainingEngine/ANSTrainingEngine.cpp @@ -17,6 +17,9 @@ #include #include #include +#include +#include +#include // DebugView: filter on "[ANSTRE]" — gated by ANSCORE_DEBUGVIEW in ANSLicense.h. @@ -164,6 +167,102 @@ namespace ANSCENTER CloseHandle(snapshot); return isRunning; } + bool ANSTRE::GetDeviceList(std::string& deviceList) { + deviceList.clear(); + const char* cmd = "nvidia-smi --query-gpu=index --format=csv,noheader"; + ANS_DBG("ANSTRE", "GetDeviceList: ENTER, cmd=\"%s\"", cmd); + // Unconditional console trace — visible without enabling the DebugView gate. + // std::endl flushes so output is seen even if the host process crashes shortly after. + std::cout << "[ANSTRE][GetDeviceList] ENTER, cmd=\"" << cmd << "\"" << std::endl; + + // `nvidia-smi --query-gpu=index --format=csv,noheader` prints one integer per + // detected GPU. If the binary is missing, cmd.exe writes an error message to + // our captured stderr instead — none of those lines parse as a bare integer, + // so the count below stays at 0 and we return false. + const std::string raw = ExecuteCommand(cmd); + ANS_DBG("ANSTRE", "GetDeviceList: nvidia-smi returned %zu bytes", raw.size()); + std::cout << "[ANSTRE][GetDeviceList] nvidia-smi returned " << raw.size() << " bytes" << std::endl; + + // Dump the raw output (newlines flattened to '|') so DebugView shows it on one + // line — invaluable for diagnosing missing driver / PATH / permission issues + // on customer machines without round-trips for rebuilds. + if (!raw.empty()) { + std::string oneLine = raw; + std::replace(oneLine.begin(), oneLine.end(), '\r', ' '); + std::replace(oneLine.begin(), oneLine.end(), '\n', '|'); + ANS_DBG("ANSTRE", "GetDeviceList: raw output: %s", oneLine.c_str()); + std::cout << "[ANSTRE][GetDeviceList] raw output: " << oneLine << std::endl; + } + + if (raw.empty()) { + ANS_DBG("ANSTRE", "GetDeviceList: EXIT false (nvidia-smi produced no output -> no NVIDIA GPU)"); + std::cout << "[ANSTRE][GetDeviceList] EXIT false (nvidia-smi produced no output -> no NVIDIA GPU)" << std::endl; + return false; + } + + std::vector indices; + std::istringstream iss(raw); + std::string line; + int lineNo = 0; + while (std::getline(iss, line)) { + ++lineNo; + // Trim whitespace (incl. CR from Windows line endings). + const auto first = line.find_first_not_of(" \t\r\n"); + if (first == std::string::npos) continue; + const auto last = line.find_last_not_of(" \t\r\n"); + const std::string trimmed = line.substr(first, last - first + 1); + if (trimmed.empty()) continue; + if (!std::all_of(trimmed.begin(), trimmed.end(), [](unsigned char c){ return std::isdigit(c) != 0; })) { + // Non-numeric line — likely an error message from cmd.exe ("'nvidia-smi' + // is not recognized...") or a driver warning. Skip it but log it so the + // reason for a no-GPU verdict is visible. + ANS_DBG("ANSTRE", "GetDeviceList: skip non-numeric line #%d: \"%s\"", lineNo, trimmed.c_str()); + std::cout << "[ANSTRE][GetDeviceList] skip non-numeric line #" << lineNo << ": \"" << trimmed << "\"" << std::endl; + continue; + } + try { + int idx = std::stoi(trimmed); + indices.push_back(idx); + ANS_DBG("ANSTRE", "GetDeviceList: parsed GPU index %d (line #%d)", idx, lineNo); + std::cout << "[ANSTRE][GetDeviceList] parsed GPU index " << idx << " (line #" << lineNo << ")" << std::endl; + } catch (const std::exception& e) { + // stoi can throw on overflow — ignore and keep parsing. + ANS_DBG("ANSTRE", "GetDeviceList: stoi failed on line #%d \"%s\": %s", lineNo, trimmed.c_str(), e.what()); + std::cout << "[ANSTRE][GetDeviceList] stoi failed on line #" << lineNo << " \"" << trimmed << "\": " << e.what() << std::endl; + } + } + + if (indices.empty()) { + ANS_DBG("ANSTRE", "GetDeviceList: EXIT false (nvidia-smi output had no numeric GPU indices -> no NVIDIA GPU)"); + std::cout << "[ANSTRE][GetDeviceList] EXIT false (nvidia-smi output had no numeric GPU indices -> no NVIDIA GPU)" << std::endl; + return false; + } + + std::ostringstream oss; + oss << "device="; + for (size_t i = 0; i < indices.size(); ++i) { + if (i > 0) oss << ","; + oss << indices[i]; + } + deviceList = oss.str(); + ANS_DBG("ANSTRE", "GetDeviceList: EXIT true, found %zu GPU(s) -> \"%s\"", indices.size(), deviceList.c_str()); + std::cout << "[ANSTRE][GetDeviceList] EXIT true, found " << indices.size() << " GPU(s) -> \"" << deviceList << "\"" << std::endl; + return true; + } + std::string ANSTRE::GetDeviceArgOrDefault() { + std::string list; + if (GetDeviceList(list)) { + const std::string arg = " " + list; + ANS_DBG("ANSTRE", "GetDeviceArgOrDefault: using \"%s\"", arg.c_str()); + std::cout << "[ANSTRE][GetDeviceArgOrDefault] using \"" << arg << "\"" << std::endl; + return arg; + } + ANS_DBG("ANSTRE", "GetDeviceArgOrDefault: no NVIDIA GPU, using \" device=-1\""); + std::cout << "[ANSTRE][GetDeviceArgOrDefault] no NVIDIA GPU, using \" device=-1\"" << std::endl; + this->_logger.LogWarn("ANSTRE::GetDeviceArgOrDefault", + "No NVIDIA GPU detected, falling back to device=-1", __FILE__, __LINE__); + return " device=-1"; + } bool ANSTRE::ZipExperiment(std::string projectName, int experimentNumber) { bool zipModelExist = false; _projectName = Space2Underscore(projectName); diff --git a/modules/ANSTrainingEngine/ANSTrainingEngine.h b/modules/ANSTrainingEngine/ANSTrainingEngine.h index e345c04..87ce915 100644 --- a/modules/ANSTrainingEngine/ANSTrainingEngine.h +++ b/modules/ANSTrainingEngine/ANSTrainingEngine.h @@ -35,6 +35,14 @@ namespace ANSCENTER { [[nodiscard]] std::string GetProjectDirectory() { return _projectDirectory; } [[nodiscard]] bool ZipExperiment(std::string projectName, int experimentNumber); [[nodiscard]] bool IsProcessRunning(const char* processName); + // Detect NVIDIA GPUs via nvidia-smi. Returns false if no NVIDIA GPU is present + // (or nvidia-smi is unavailable). On success, populates deviceList with a yolo + // CLI-compatible token, e.g. "device=0", "device=0,1", "device=0,1,2,3,4,5". + [[nodiscard]] bool GetDeviceList(std::string& deviceList); + // Yolo-CLI-ready device argument with a leading space, suitable for direct + // concatenation into a yolo command line. Returns " device=0,1,..." when GPUs + // are detected, or " device=-1" as a fallback when none are. + [[nodiscard]] std::string GetDeviceArgOrDefault(); [[nodiscard]] double ParseObjectDetectionMAP(std::string stEvaluationResult); [[nodiscard]] std::string GetLatestValidFolderAndDeleteOthers(const std::string& projectName, const std::string& rootPath); protected: diff --git a/modules/ANSTrainingEngine/ANSYOLOCLTrainingEngine.cpp b/modules/ANSTrainingEngine/ANSYOLOCLTrainingEngine.cpp index 33738df..c26171b 100644 --- a/modules/ANSTrainingEngine/ANSYOLOCLTrainingEngine.cpp +++ b/modules/ANSTrainingEngine/ANSYOLOCLTrainingEngine.cpp @@ -425,7 +425,7 @@ namespace ANSCENTER " data=" + "\"" + projectDataPath + "\"" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " plots=False" + " patience=0" + @@ -523,7 +523,7 @@ namespace ANSCENTER " data=" + "\"" + projectDataPath + "\"" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " plots=False" " patience=0" + @@ -596,7 +596,7 @@ namespace ANSCENTER " split=val" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " mlflow=False" + @@ -1124,7 +1124,7 @@ namespace ANSCENTER " split=val" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " mlflow=False" + diff --git a/modules/ANSTrainingEngine/ANSYOLOOBBTrainingEngine.cpp b/modules/ANSTrainingEngine/ANSYOLOOBBTrainingEngine.cpp index 19ce2c9..8600192 100644 --- a/modules/ANSTrainingEngine/ANSYOLOOBBTrainingEngine.cpp +++ b/modules/ANSTrainingEngine/ANSYOLOOBBTrainingEngine.cpp @@ -433,7 +433,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " patience=0" + " plots=False" + @@ -533,7 +533,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " patience=0" + " plots=False" + @@ -607,7 +607,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " save_json=False" + @@ -1213,7 +1213,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " save_json=False" + diff --git a/modules/ANSTrainingEngine/ANSYOLOSEGTrainingEngine.cpp b/modules/ANSTrainingEngine/ANSYOLOSEGTrainingEngine.cpp index ac224aa..3a43d8e 100644 --- a/modules/ANSTrainingEngine/ANSYOLOSEGTrainingEngine.cpp +++ b/modules/ANSTrainingEngine/ANSYOLOSEGTrainingEngine.cpp @@ -432,7 +432,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " patience=0" + " plots=False" + @@ -531,7 +531,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " patience=0" + " plots=False" + @@ -602,7 +602,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " save_json=False" + @@ -1207,7 +1207,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " save_json=False" + diff --git a/modules/ANSTrainingEngine/ANSYOLOTrainingEngine.cpp b/modules/ANSTrainingEngine/ANSYOLOTrainingEngine.cpp index ae7aaef..949c68a 100644 --- a/modules/ANSTrainingEngine/ANSYOLOTrainingEngine.cpp +++ b/modules/ANSTrainingEngine/ANSYOLOTrainingEngine.cpp @@ -457,13 +457,13 @@ namespace ANSCENTER learningRate = cfgMapData.at("lr0"); batch = cfgMapData.at("batch"); } - std::string trainCommand = trainScript + + std::string trainCommand = trainScript + " detect train model=" + "\"" + preTrainWeightFile + "\"" + " data=" + "\"" + projectDataPath + "\"" + " imgsz=640" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " patience=0" + //" lr0= 0.01" + @@ -573,7 +573,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=" + batch + - " device=0" + + GetDeviceArgOrDefault() + " epochs=" + epochs + " patience=0" + //" lr0= 0.01" + @@ -649,7 +649,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " save_json=False" + @@ -1265,7 +1265,7 @@ namespace ANSCENTER " imgsz=640" + " workers=8" + " batch=8" + - " device=0" + + GetDeviceArgOrDefault() + " conf=0.25" + " plots=False" + " save_json=False" + diff --git a/tests/ANSODTRE-UnitTest/ANSODTRE-UnitTest.cpp b/tests/ANSODTRE-UnitTest/ANSODTRE-UnitTest.cpp index ec7e519..28f85f7 100644 --- a/tests/ANSODTRE-UnitTest/ANSODTRE-UnitTest.cpp +++ b/tests/ANSODTRE-UnitTest/ANSODTRE-UnitTest.cpp @@ -438,47 +438,47 @@ int TestSegmentationEngine() {// For New Yolo version } //// Set the working directory - bool setWorkingDirectory = ansEngine.SetWorkingDirectory("C:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo"); + bool setWorkingDirectory = ansEngine.SetWorkingDirectory("E:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo"); if (!setWorkingDirectory)std::cout << "Working Directory Not Set" << std::endl; ////1. Create a project - bool createdProject = ansEngine.CreateProject("SegmentationTest"); + bool createdProject = ansEngine.CreateProject("SegmentationTest1"); if (!createdProject)std::cout << "Project Not Created" << std::endl; ////2. Browse to the working directory projectDirectory = ansEngine.GetProjectDirectory(); std::cout << "Project Directory:" << projectDirectory << std::endl; - bool dataUploaded = ansEngine.UploadTrainingData("SegmentationTest"); + bool dataUploaded = ansEngine.UploadTrainingData("SegmentationTest1"); if (!dataUploaded)std::cout << "Cannot upload data" << std::endl; //////3. Create a training engine - bool createEngine = ansEngine.CreateTrainingEngine("SegmentationTest", 1, 0, 2000, 32, 0.001); + bool createEngine = ansEngine.CreateTrainingEngine("SegmentationTest1", 1, 0, 2000, 32, 0.001); if (!createEngine)std::cout << "Cannot create engine" << std::endl; - std::string projectExperimentStatus = ansEngine.GetProjectExperimentStatus("SegmentationTest", 1); + std::string projectExperimentStatus = ansEngine.GetProjectExperimentStatus("SegmentationTest1", 1); std::cout << projectExperimentStatus << std::endl; - std::string trainingCommand = ansEngine.GenerateTrainingCommand("SegmentationTest", 1, 0); + std::string trainingCommand = ansEngine.GenerateTrainingCommand("SegmentationTest1", 1, 0); std::cout << "Training Command:" << trainingCommand << std::endl; //// 4. Evaluate the model - std::string evaluateModelCommand = ansEngine.EvaluateModel("SegmentationTest", 1); + std::string evaluateModelCommand = ansEngine.EvaluateModel("SegmentationTest1", 1); std::cout << "Evaluation Result:" << evaluateModelCommand << std::endl; //// 5. Download the model - std::string saveModelPath = "C:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo\\Models"; - ansEngine.DownloadModel("SegmentationTest", 1, saveModelPath, 0); - ansEngine.DownloadModel("SegmentationTest", 1, saveModelPath, 1); + std::string saveModelPath = "E:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo\\Models"; + ansEngine.DownloadModel("SegmentationTest1", 1, saveModelPath, 0); + ansEngine.DownloadModel("SegmentationTest1", 1, saveModelPath, 1); return 0; } int main() { - TestSegmentationEngine(); + TestSegmentationEngine(); //TestANSYLTRE(); //TestANSYLCLTRE(); //FallDetection();