Support multiple GPUs

This commit is contained in:
2026-05-02 12:24:53 +10:00
parent 8359511895
commit cff2a90fad
7 changed files with 135 additions and 28 deletions
@@ -17,6 +17,9 @@
#include <tlhelp32.h>
#include <unordered_map>
#include <condition_variable>
#include <sstream>
#include <algorithm>
#include <cctype>
// DebugView: filter on "[ANSTRE]" — gated by ANSCORE_DEBUGVIEW in ANSLicense.h.
@@ -164,6 +167,102 @@ namespace ANSCENTER
CloseHandle(snapshot);
return isRunning;
}
bool ANSTRE::GetDeviceList(std::string& deviceList) {
deviceList.clear();
const char* cmd = "nvidia-smi --query-gpu=index --format=csv,noheader";
ANS_DBG("ANSTRE", "GetDeviceList: ENTER, cmd=\"%s\"", cmd);
// Unconditional console trace — visible without enabling the DebugView gate.
// std::endl flushes so output is seen even if the host process crashes shortly after.
std::cout << "[ANSTRE][GetDeviceList] ENTER, cmd=\"" << cmd << "\"" << std::endl;
// `nvidia-smi --query-gpu=index --format=csv,noheader` prints one integer per
// detected GPU. If the binary is missing, cmd.exe writes an error message to
// our captured stderr instead — none of those lines parse as a bare integer,
// so the count below stays at 0 and we return false.
const std::string raw = ExecuteCommand(cmd);
ANS_DBG("ANSTRE", "GetDeviceList: nvidia-smi returned %zu bytes", raw.size());
std::cout << "[ANSTRE][GetDeviceList] nvidia-smi returned " << raw.size() << " bytes" << std::endl;
// Dump the raw output (newlines flattened to '|') so DebugView shows it on one
// line — invaluable for diagnosing missing driver / PATH / permission issues
// on customer machines without round-trips for rebuilds.
if (!raw.empty()) {
std::string oneLine = raw;
std::replace(oneLine.begin(), oneLine.end(), '\r', ' ');
std::replace(oneLine.begin(), oneLine.end(), '\n', '|');
ANS_DBG("ANSTRE", "GetDeviceList: raw output: %s", oneLine.c_str());
std::cout << "[ANSTRE][GetDeviceList] raw output: " << oneLine << std::endl;
}
if (raw.empty()) {
ANS_DBG("ANSTRE", "GetDeviceList: EXIT false (nvidia-smi produced no output -> no NVIDIA GPU)");
std::cout << "[ANSTRE][GetDeviceList] EXIT false (nvidia-smi produced no output -> no NVIDIA GPU)" << std::endl;
return false;
}
std::vector<int> indices;
std::istringstream iss(raw);
std::string line;
int lineNo = 0;
while (std::getline(iss, line)) {
++lineNo;
// Trim whitespace (incl. CR from Windows line endings).
const auto first = line.find_first_not_of(" \t\r\n");
if (first == std::string::npos) continue;
const auto last = line.find_last_not_of(" \t\r\n");
const std::string trimmed = line.substr(first, last - first + 1);
if (trimmed.empty()) continue;
if (!std::all_of(trimmed.begin(), trimmed.end(), [](unsigned char c){ return std::isdigit(c) != 0; })) {
// Non-numeric line — likely an error message from cmd.exe ("'nvidia-smi'
// is not recognized...") or a driver warning. Skip it but log it so the
// reason for a no-GPU verdict is visible.
ANS_DBG("ANSTRE", "GetDeviceList: skip non-numeric line #%d: \"%s\"", lineNo, trimmed.c_str());
std::cout << "[ANSTRE][GetDeviceList] skip non-numeric line #" << lineNo << ": \"" << trimmed << "\"" << std::endl;
continue;
}
try {
int idx = std::stoi(trimmed);
indices.push_back(idx);
ANS_DBG("ANSTRE", "GetDeviceList: parsed GPU index %d (line #%d)", idx, lineNo);
std::cout << "[ANSTRE][GetDeviceList] parsed GPU index " << idx << " (line #" << lineNo << ")" << std::endl;
} catch (const std::exception& e) {
// stoi can throw on overflow — ignore and keep parsing.
ANS_DBG("ANSTRE", "GetDeviceList: stoi failed on line #%d \"%s\": %s", lineNo, trimmed.c_str(), e.what());
std::cout << "[ANSTRE][GetDeviceList] stoi failed on line #" << lineNo << " \"" << trimmed << "\": " << e.what() << std::endl;
}
}
if (indices.empty()) {
ANS_DBG("ANSTRE", "GetDeviceList: EXIT false (nvidia-smi output had no numeric GPU indices -> no NVIDIA GPU)");
std::cout << "[ANSTRE][GetDeviceList] EXIT false (nvidia-smi output had no numeric GPU indices -> no NVIDIA GPU)" << std::endl;
return false;
}
std::ostringstream oss;
oss << "device=";
for (size_t i = 0; i < indices.size(); ++i) {
if (i > 0) oss << ",";
oss << indices[i];
}
deviceList = oss.str();
ANS_DBG("ANSTRE", "GetDeviceList: EXIT true, found %zu GPU(s) -> \"%s\"", indices.size(), deviceList.c_str());
std::cout << "[ANSTRE][GetDeviceList] EXIT true, found " << indices.size() << " GPU(s) -> \"" << deviceList << "\"" << std::endl;
return true;
}
std::string ANSTRE::GetDeviceArgOrDefault() {
std::string list;
if (GetDeviceList(list)) {
const std::string arg = " " + list;
ANS_DBG("ANSTRE", "GetDeviceArgOrDefault: using \"%s\"", arg.c_str());
std::cout << "[ANSTRE][GetDeviceArgOrDefault] using \"" << arg << "\"" << std::endl;
return arg;
}
ANS_DBG("ANSTRE", "GetDeviceArgOrDefault: no NVIDIA GPU, using \" device=-1\"");
std::cout << "[ANSTRE][GetDeviceArgOrDefault] no NVIDIA GPU, using \" device=-1\"" << std::endl;
this->_logger.LogWarn("ANSTRE::GetDeviceArgOrDefault",
"No NVIDIA GPU detected, falling back to device=-1", __FILE__, __LINE__);
return " device=-1";
}
bool ANSTRE::ZipExperiment(std::string projectName, int experimentNumber) {
bool zipModelExist = false;
_projectName = Space2Underscore(projectName);