Support multiple GPUs
This commit is contained in:
@@ -17,6 +17,9 @@
|
|||||||
#include <tlhelp32.h>
|
#include <tlhelp32.h>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
|
#include <sstream>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cctype>
|
||||||
|
|
||||||
// DebugView: filter on "[ANSTRE]" — gated by ANSCORE_DEBUGVIEW in ANSLicense.h.
|
// DebugView: filter on "[ANSTRE]" — gated by ANSCORE_DEBUGVIEW in ANSLicense.h.
|
||||||
|
|
||||||
@@ -164,6 +167,102 @@ namespace ANSCENTER
|
|||||||
CloseHandle(snapshot);
|
CloseHandle(snapshot);
|
||||||
return isRunning;
|
return isRunning;
|
||||||
}
|
}
|
||||||
|
bool ANSTRE::GetDeviceList(std::string& deviceList) {
|
||||||
|
deviceList.clear();
|
||||||
|
const char* cmd = "nvidia-smi --query-gpu=index --format=csv,noheader";
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: ENTER, cmd=\"%s\"", cmd);
|
||||||
|
// Unconditional console trace — visible without enabling the DebugView gate.
|
||||||
|
// std::endl flushes so output is seen even if the host process crashes shortly after.
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] ENTER, cmd=\"" << cmd << "\"" << std::endl;
|
||||||
|
|
||||||
|
// `nvidia-smi --query-gpu=index --format=csv,noheader` prints one integer per
|
||||||
|
// detected GPU. If the binary is missing, cmd.exe writes an error message to
|
||||||
|
// our captured stderr instead — none of those lines parse as a bare integer,
|
||||||
|
// so the count below stays at 0 and we return false.
|
||||||
|
const std::string raw = ExecuteCommand(cmd);
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: nvidia-smi returned %zu bytes", raw.size());
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] nvidia-smi returned " << raw.size() << " bytes" << std::endl;
|
||||||
|
|
||||||
|
// Dump the raw output (newlines flattened to '|') so DebugView shows it on one
|
||||||
|
// line — invaluable for diagnosing missing driver / PATH / permission issues
|
||||||
|
// on customer machines without round-trips for rebuilds.
|
||||||
|
if (!raw.empty()) {
|
||||||
|
std::string oneLine = raw;
|
||||||
|
std::replace(oneLine.begin(), oneLine.end(), '\r', ' ');
|
||||||
|
std::replace(oneLine.begin(), oneLine.end(), '\n', '|');
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: raw output: %s", oneLine.c_str());
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] raw output: " << oneLine << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (raw.empty()) {
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: EXIT false (nvidia-smi produced no output -> no NVIDIA GPU)");
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] EXIT false (nvidia-smi produced no output -> no NVIDIA GPU)" << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<int> indices;
|
||||||
|
std::istringstream iss(raw);
|
||||||
|
std::string line;
|
||||||
|
int lineNo = 0;
|
||||||
|
while (std::getline(iss, line)) {
|
||||||
|
++lineNo;
|
||||||
|
// Trim whitespace (incl. CR from Windows line endings).
|
||||||
|
const auto first = line.find_first_not_of(" \t\r\n");
|
||||||
|
if (first == std::string::npos) continue;
|
||||||
|
const auto last = line.find_last_not_of(" \t\r\n");
|
||||||
|
const std::string trimmed = line.substr(first, last - first + 1);
|
||||||
|
if (trimmed.empty()) continue;
|
||||||
|
if (!std::all_of(trimmed.begin(), trimmed.end(), [](unsigned char c){ return std::isdigit(c) != 0; })) {
|
||||||
|
// Non-numeric line — likely an error message from cmd.exe ("'nvidia-smi'
|
||||||
|
// is not recognized...") or a driver warning. Skip it but log it so the
|
||||||
|
// reason for a no-GPU verdict is visible.
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: skip non-numeric line #%d: \"%s\"", lineNo, trimmed.c_str());
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] skip non-numeric line #" << lineNo << ": \"" << trimmed << "\"" << std::endl;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
int idx = std::stoi(trimmed);
|
||||||
|
indices.push_back(idx);
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: parsed GPU index %d (line #%d)", idx, lineNo);
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] parsed GPU index " << idx << " (line #" << lineNo << ")" << std::endl;
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
// stoi can throw on overflow — ignore and keep parsing.
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: stoi failed on line #%d \"%s\": %s", lineNo, trimmed.c_str(), e.what());
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] stoi failed on line #" << lineNo << " \"" << trimmed << "\": " << e.what() << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (indices.empty()) {
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: EXIT false (nvidia-smi output had no numeric GPU indices -> no NVIDIA GPU)");
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] EXIT false (nvidia-smi output had no numeric GPU indices -> no NVIDIA GPU)" << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::ostringstream oss;
|
||||||
|
oss << "device=";
|
||||||
|
for (size_t i = 0; i < indices.size(); ++i) {
|
||||||
|
if (i > 0) oss << ",";
|
||||||
|
oss << indices[i];
|
||||||
|
}
|
||||||
|
deviceList = oss.str();
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceList: EXIT true, found %zu GPU(s) -> \"%s\"", indices.size(), deviceList.c_str());
|
||||||
|
std::cout << "[ANSTRE][GetDeviceList] EXIT true, found " << indices.size() << " GPU(s) -> \"" << deviceList << "\"" << std::endl;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
std::string ANSTRE::GetDeviceArgOrDefault() {
|
||||||
|
std::string list;
|
||||||
|
if (GetDeviceList(list)) {
|
||||||
|
const std::string arg = " " + list;
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceArgOrDefault: using \"%s\"", arg.c_str());
|
||||||
|
std::cout << "[ANSTRE][GetDeviceArgOrDefault] using \"" << arg << "\"" << std::endl;
|
||||||
|
return arg;
|
||||||
|
}
|
||||||
|
ANS_DBG("ANSTRE", "GetDeviceArgOrDefault: no NVIDIA GPU, using \" device=-1\"");
|
||||||
|
std::cout << "[ANSTRE][GetDeviceArgOrDefault] no NVIDIA GPU, using \" device=-1\"" << std::endl;
|
||||||
|
this->_logger.LogWarn("ANSTRE::GetDeviceArgOrDefault",
|
||||||
|
"No NVIDIA GPU detected, falling back to device=-1", __FILE__, __LINE__);
|
||||||
|
return " device=-1";
|
||||||
|
}
|
||||||
bool ANSTRE::ZipExperiment(std::string projectName, int experimentNumber) {
|
bool ANSTRE::ZipExperiment(std::string projectName, int experimentNumber) {
|
||||||
bool zipModelExist = false;
|
bool zipModelExist = false;
|
||||||
_projectName = Space2Underscore(projectName);
|
_projectName = Space2Underscore(projectName);
|
||||||
|
|||||||
@@ -35,6 +35,14 @@ namespace ANSCENTER {
|
|||||||
[[nodiscard]] std::string GetProjectDirectory() { return _projectDirectory; }
|
[[nodiscard]] std::string GetProjectDirectory() { return _projectDirectory; }
|
||||||
[[nodiscard]] bool ZipExperiment(std::string projectName, int experimentNumber);
|
[[nodiscard]] bool ZipExperiment(std::string projectName, int experimentNumber);
|
||||||
[[nodiscard]] bool IsProcessRunning(const char* processName);
|
[[nodiscard]] bool IsProcessRunning(const char* processName);
|
||||||
|
// Detect NVIDIA GPUs via nvidia-smi. Returns false if no NVIDIA GPU is present
|
||||||
|
// (or nvidia-smi is unavailable). On success, populates deviceList with a yolo
|
||||||
|
// CLI-compatible token, e.g. "device=0", "device=0,1", "device=0,1,2,3,4,5".
|
||||||
|
[[nodiscard]] bool GetDeviceList(std::string& deviceList);
|
||||||
|
// Yolo-CLI-ready device argument with a leading space, suitable for direct
|
||||||
|
// concatenation into a yolo command line. Returns " device=0,1,..." when GPUs
|
||||||
|
// are detected, or " device=-1" as a fallback when none are.
|
||||||
|
[[nodiscard]] std::string GetDeviceArgOrDefault();
|
||||||
[[nodiscard]] double ParseObjectDetectionMAP(std::string stEvaluationResult);
|
[[nodiscard]] double ParseObjectDetectionMAP(std::string stEvaluationResult);
|
||||||
[[nodiscard]] std::string GetLatestValidFolderAndDeleteOthers(const std::string& projectName, const std::string& rootPath);
|
[[nodiscard]] std::string GetLatestValidFolderAndDeleteOthers(const std::string& projectName, const std::string& rootPath);
|
||||||
protected:
|
protected:
|
||||||
|
|||||||
@@ -425,7 +425,7 @@ namespace ANSCENTER
|
|||||||
" data=" + "\"" + projectDataPath + "\"" +
|
" data=" + "\"" + projectDataPath + "\"" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
@@ -523,7 +523,7 @@ namespace ANSCENTER
|
|||||||
" data=" + "\"" + projectDataPath + "\"" +
|
" data=" + "\"" + projectDataPath + "\"" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" plots=False"
|
" plots=False"
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
@@ -596,7 +596,7 @@ namespace ANSCENTER
|
|||||||
" split=val" +
|
" split=val" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" mlflow=False" +
|
" mlflow=False" +
|
||||||
@@ -1124,7 +1124,7 @@ namespace ANSCENTER
|
|||||||
" split=val" +
|
" split=val" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" mlflow=False" +
|
" mlflow=False" +
|
||||||
|
|||||||
@@ -433,7 +433,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
@@ -533,7 +533,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
@@ -607,7 +607,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" save_json=False" +
|
" save_json=False" +
|
||||||
@@ -1213,7 +1213,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" save_json=False" +
|
" save_json=False" +
|
||||||
|
|||||||
@@ -432,7 +432,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
@@ -531,7 +531,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
@@ -602,7 +602,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" save_json=False" +
|
" save_json=False" +
|
||||||
@@ -1207,7 +1207,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" save_json=False" +
|
" save_json=False" +
|
||||||
|
|||||||
@@ -457,13 +457,13 @@ namespace ANSCENTER
|
|||||||
learningRate = cfgMapData.at("lr0");
|
learningRate = cfgMapData.at("lr0");
|
||||||
batch = cfgMapData.at("batch");
|
batch = cfgMapData.at("batch");
|
||||||
}
|
}
|
||||||
std::string trainCommand = trainScript +
|
std::string trainCommand = trainScript +
|
||||||
" detect train model=" + "\"" + preTrainWeightFile + "\"" +
|
" detect train model=" + "\"" + preTrainWeightFile + "\"" +
|
||||||
" data=" + "\"" + projectDataPath + "\"" +
|
" data=" + "\"" + projectDataPath + "\"" +
|
||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
//" lr0= 0.01" +
|
//" lr0= 0.01" +
|
||||||
@@ -573,7 +573,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=" + batch +
|
" batch=" + batch +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" epochs=" + epochs +
|
" epochs=" + epochs +
|
||||||
" patience=0" +
|
" patience=0" +
|
||||||
//" lr0= 0.01" +
|
//" lr0= 0.01" +
|
||||||
@@ -649,7 +649,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" save_json=False" +
|
" save_json=False" +
|
||||||
@@ -1265,7 +1265,7 @@ namespace ANSCENTER
|
|||||||
" imgsz=640" +
|
" imgsz=640" +
|
||||||
" workers=8" +
|
" workers=8" +
|
||||||
" batch=8" +
|
" batch=8" +
|
||||||
" device=0" +
|
GetDeviceArgOrDefault() +
|
||||||
" conf=0.25" +
|
" conf=0.25" +
|
||||||
" plots=False" +
|
" plots=False" +
|
||||||
" save_json=False" +
|
" save_json=False" +
|
||||||
|
|||||||
@@ -438,47 +438,47 @@ int TestSegmentationEngine() {// For New Yolo version
|
|||||||
}
|
}
|
||||||
|
|
||||||
//// Set the working directory
|
//// Set the working directory
|
||||||
bool setWorkingDirectory = ansEngine.SetWorkingDirectory("C:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo");
|
bool setWorkingDirectory = ansEngine.SetWorkingDirectory("E:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo");
|
||||||
if (!setWorkingDirectory)std::cout << "Working Directory Not Set" << std::endl;
|
if (!setWorkingDirectory)std::cout << "Working Directory Not Set" << std::endl;
|
||||||
|
|
||||||
|
|
||||||
////1. Create a project
|
////1. Create a project
|
||||||
bool createdProject = ansEngine.CreateProject("SegmentationTest");
|
bool createdProject = ansEngine.CreateProject("SegmentationTest1");
|
||||||
if (!createdProject)std::cout << "Project Not Created" << std::endl;
|
if (!createdProject)std::cout << "Project Not Created" << std::endl;
|
||||||
|
|
||||||
////2. Browse to the working directory
|
////2. Browse to the working directory
|
||||||
projectDirectory = ansEngine.GetProjectDirectory();
|
projectDirectory = ansEngine.GetProjectDirectory();
|
||||||
std::cout << "Project Directory:" << projectDirectory << std::endl;
|
std::cout << "Project Directory:" << projectDirectory << std::endl;
|
||||||
|
|
||||||
bool dataUploaded = ansEngine.UploadTrainingData("SegmentationTest");
|
bool dataUploaded = ansEngine.UploadTrainingData("SegmentationTest1");
|
||||||
if (!dataUploaded)std::cout << "Cannot upload data" << std::endl;
|
if (!dataUploaded)std::cout << "Cannot upload data" << std::endl;
|
||||||
|
|
||||||
//////3. Create a training engine
|
//////3. Create a training engine
|
||||||
|
|
||||||
bool createEngine = ansEngine.CreateTrainingEngine("SegmentationTest", 1, 0, 2000, 32, 0.001);
|
bool createEngine = ansEngine.CreateTrainingEngine("SegmentationTest1", 1, 0, 2000, 32, 0.001);
|
||||||
if (!createEngine)std::cout << "Cannot create engine" << std::endl;
|
if (!createEngine)std::cout << "Cannot create engine" << std::endl;
|
||||||
|
|
||||||
std::string projectExperimentStatus = ansEngine.GetProjectExperimentStatus("SegmentationTest", 1);
|
std::string projectExperimentStatus = ansEngine.GetProjectExperimentStatus("SegmentationTest1", 1);
|
||||||
std::cout << projectExperimentStatus << std::endl;
|
std::cout << projectExperimentStatus << std::endl;
|
||||||
|
|
||||||
std::string trainingCommand = ansEngine.GenerateTrainingCommand("SegmentationTest", 1, 0);
|
std::string trainingCommand = ansEngine.GenerateTrainingCommand("SegmentationTest1", 1, 0);
|
||||||
std::cout << "Training Command:" << trainingCommand << std::endl;
|
std::cout << "Training Command:" << trainingCommand << std::endl;
|
||||||
|
|
||||||
//// 4. Evaluate the model
|
//// 4. Evaluate the model
|
||||||
std::string evaluateModelCommand = ansEngine.EvaluateModel("SegmentationTest", 1);
|
std::string evaluateModelCommand = ansEngine.EvaluateModel("SegmentationTest1", 1);
|
||||||
std::cout << "Evaluation Result:" << evaluateModelCommand << std::endl;
|
std::cout << "Evaluation Result:" << evaluateModelCommand << std::endl;
|
||||||
|
|
||||||
//// 5. Download the model
|
//// 5. Download the model
|
||||||
std::string saveModelPath = "C:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo\\Models";
|
std::string saveModelPath = "E:\\Programs\\TrainingToolTest\\SegmentationEngine\\SegmeTxtYolo\\Models";
|
||||||
ansEngine.DownloadModel("SegmentationTest", 1, saveModelPath, 0);
|
ansEngine.DownloadModel("SegmentationTest1", 1, saveModelPath, 0);
|
||||||
ansEngine.DownloadModel("SegmentationTest", 1, saveModelPath, 1);
|
ansEngine.DownloadModel("SegmentationTest1", 1, saveModelPath, 1);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main()
|
int main()
|
||||||
{
|
{
|
||||||
TestSegmentationEngine();
|
TestSegmentationEngine();
|
||||||
//TestANSYLTRE();
|
//TestANSYLTRE();
|
||||||
//TestANSYLCLTRE();
|
//TestANSYLCLTRE();
|
||||||
//FallDetection();
|
//FallDetection();
|
||||||
|
|||||||
Reference in New Issue
Block a user