#include "ANSYOLOOBBTrainingEngine.h" #include "Utility.h" #include #include #include #include namespace ANSCENTER { ANYLOBBTRE::ANYLOBBTRE() { _projectDirectory = "C:\\ProgramData\\Sh7O7nUe7vJ\\ansobb"; _engineDirectory = "C:\\ProgramData\\ANSCENTER\\Python\\Scripts"; _modelTemplateDirectory = "C:\\ProgramData\\ANSCENTER\\Shared"; _currentExperimentNumber = 1; _isLicenseValid = false; _licenseKey = ""; _zipPassword = ""; _changeProjectDirectory = false; _isLatestEngine = true;// Default is YoloV10 engine } ANYLOBBTRE::~ANYLOBBTRE() { _projects.clear(); _zipPassword = ""; _isLicenseValid = false; _changeProjectDirectory = false; _isLatestEngine = true; } void ANYLOBBTRE::CheckLicense() { try { _isLicenseValid = ANSCENTER::ANSLicenseHelper::LicenseVerification(_licenseKey, 1008, "ANSTS");//Default productId=1008 (ANSTS) } catch (std::exception& e) { this->_logger.LogFatal("ANYLOBBTRE::CheckLicense()", e.what(), __FILE__, __LINE__); } } bool ANYLOBBTRE::Init(std::string licenseKey, std::string projectDirectory, std::string engineDirectory, std::string modelTemplateDirectory, std::string modelZipPassword, bool isLatestEngine) { _licenseKey = licenseKey; _isLatestEngine = isLatestEngine; CheckLicense(); if (!_isLicenseValid) { this->_logger.LogError("ANYLOBBTRE::Init", "Invalid license key", __FILE__, __LINE__); return false; } if (projectDirectory.empty())_projectDirectory = "C:\\ProgramData\\Sh7O7nUe7vJ\\ansobb"; else _projectDirectory = projectDirectory + "\\ansobb"; _engineDirectory = engineDirectory; _modelTemplateDirectory = modelTemplateDirectory; if (_engineDirectory.empty())_engineDirectory = "C:\\ProgramData\\ANSCENTER\\Python311\\Scripts"; if (_modelTemplateDirectory.empty())_modelTemplateDirectory = "C:\\ProgramData\\ANSCENTER\\Python311\\Lib\\site-packages\\data"; // Change the project directory to 1 level down if there is 1 folder in the project directory if (!FolderExist(_projectDirectory)) fs::create_directory(_projectDirectory); _modelZipPassword = modelZipPassword; _zipPassword = "Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag=="; _modelZipPassword = _zipPassword; // Some logging in here std::string _ansdnnEngine = _engineDirectory + "\\yolo.exe"; if (!FileExist(_ansdnnEngine))this->_logger.LogError("ANYLOBBTRE::Init", "ANSDL Engine is not installed. Please install ANS Runtime Engine", __FILE__, __LINE__); std::string fastWeight = _modelTemplateDirectory + "\\ob\\ob8n.pt"; std::string accurateWeight = _modelTemplateDirectory + "\\ob\\ob8m.pt"; std::string veryAccurateWeight = _modelTemplateDirectory + "\\ob\\ob8x.pt"; yolo11nModelFile = _modelTemplateDirectory + "\\yolo11n.pt"; if (!FileExist(fastWeight))this->_logger.LogError("ANYLOBBTRE::Init", "FAST model file is missing", __FILE__, __LINE__); if (!FileExist(accurateWeight))this->_logger.LogError("ANYLOBBTRE::Init", "ACCURATE model file is missing", __FILE__, __LINE__); if (!FileExist(veryAccurateWeight))this->_logger.LogError("ANYLOBBTRE::Init", "VERYACCURATE model file is missing", __FILE__, __LINE__); if (!FileExist(yolo11nModelFile))this->_logger.LogError("ANYLOBBTRE::Init", "Evaluate model file is missing", __FILE__, __LINE__); if (FolderExist(_projectDirectory) && FolderExist(_engineDirectory) && FolderExist(_modelTemplateDirectory))return true; return false; } std::vector ANYLOBBTRE::GetProjects() { _projects.clear(); std::vector _projectNames; for (auto& entry : std::filesystem::directory_iterator(_projectDirectory)) { if (entry.is_directory()) { _projects.push_back(entry.path().string()); _projectNames.push_back(ExtractFolderName(entry.path().string())); } } return _projectNames; } std::vector ANYLOBBTRE::GetProjectExperiments(std::string projectName) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); std::string experimentPath = projectPath + "\\experiments"; if (FolderExist(experimentPath)) { std::vector files; std::vector experiments; for (const auto& entry : std::filesystem::directory_iterator(experimentPath)) { if (entry.is_regular_file() && entry.path().extension() == ".zip") { experiments.push_back(GetFileNameWithoutExtension(entry.path().string())); } } return experiments; } else { std::vector experiments; return experiments; } } std::string ANYLOBBTRE::GetProjectExperimentStatus(std::string projectName, int experimentNumber) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { std::string experimentFolder = projectPath + "\\experiments"; if (FolderExist(experimentFolder)) { std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber) + ".zip"; if (FileExist(experimentPath)) { return "TRAINED"; } else { std::string configurationPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber) + "\\train.cfg"; if (FileExist(configurationPath)) { std::string tempWeightFile = projectPath + "\\experiments\\" + std::to_string(experimentNumber) + "\\" + _projectName + "\\weights\\best.pt"; if (FileExist(tempWeightFile)) return "RUNNING"; else return "CONFIGURED"; } else return "UPLOADED"; } } else { std::string dataPath = projectPath + "\\data"; if (FileExist(dataPath)) { return "CREATED"; } else { return "UPLOADED"; } } } else return "NOTFOUND"; } bool ANYLOBBTRE::CreateProject(std::string projectName) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (projectPath.empty()) { std::string newProjectPath = _projectDirectory + "\\" + _projectName; if (FolderExist(newProjectPath))return false; fs::create_directory(newProjectPath); _projects.push_back(newProjectPath); return true; } return false; } bool ANYLOBBTRE::DeleteProject(std::string projectName) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { return DeleteFolder(projectPath); } return false; } bool ANYLOBBTRE::SetWorkingDirectory(std::string workingDirectory) { if (FolderExist(workingDirectory)) { size_t lastSlashPos = workingDirectory.find_last_of("\\/"); if (lastSlashPos != std::string::npos && workingDirectory.substr(lastSlashPos + 1) == "data") { workingDirectory = workingDirectory.substr(0, lastSlashPos); } _workingDirectory = workingDirectory; std::string dataPath = _workingDirectory + "\\data"; std::string pbLabelPath = dataPath + "\\label_map.pbtxt"; if (FileExist(pbLabelPath)) return true; else false; } return false; } int ANYLOBBTRE::GenerateClassNamesFile(std::string labelMapFile, std::string objectDataFile, bool useANSClassFile) { if (!FileExist(labelMapFile)) return 0; if (useANSClassFile) { // Use the ANS Class file std::vector classVec = ParseANSClass(labelMapFile); if (classVec.empty()) return 0; std::ofstream outFile(objectDataFile); std::string line; int fileLine = 0; for (auto& item : classVec) { outFile << item << "\n"; fileLine++; } outFile.close(); return fileLine; } else { std::ifstream inFile(labelMapFile); std::ofstream outFile(objectDataFile); if (!inFile.is_open() || !outFile.is_open()) { return 0; } std::string line; // Updated pattern: allows optional spaces between name and the colon std::regex namePattern(R"(name\s*:\s*'(.*)')"); int fileLine = 0; while (std::getline(inFile, line)) { std::smatch matches; std::cout << line << std::endl; if (std::regex_search(line, matches, namePattern) && matches.size() > 1) { outFile << matches[1].str() << "\n"; fileLine++; } } inFile.close(); outFile.close(); return fileLine; } } // We need to create a data folder in following structure // data // -images // - train // - test // - val // -labels // - train // - test // - val // -data.yaml (data.yaml) // path: sourceDataPath // train: images\train // val : images\val // names : [plate] // - classes.names bool ANYLOBBTRE::UploadTrainingData(std::string projectName) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); //1. Check if the project folder exist, then create a data folder std::string dataPath = projectPath + "\\data"; if (FolderExist(dataPath))DeleteFolder(dataPath); fs::create_directory(dataPath); std::string sourceDataPath = _workingDirectory + "\\data"; bool useANSClassFile = true; std::string sourceLabelPath = sourceDataPath + "\\ANS_Class.json"; if (!FileExist(sourceLabelPath)) { sourceLabelPath = sourceDataPath + "\\label_map.pbtxt"; useANSClassFile = false; } if (!FileExist(sourceLabelPath)) return false; //2. Generate the training dataset GenerateTrainDataSet(sourceDataPath, dataPath + "\\images", dataPath + "\\labels", 0.8); //3. Generate the classes.names file std::string classesNamesFile = dataPath + "\\classes.names"; GenerateClassNamesFile(sourceLabelPath, classesNamesFile, useANSClassFile); //4. Generate the data.yaml file std::string YalmDataFile = dataPath + "\\data.yaml"; GenerateObjectDataFile(sourceLabelPath, YalmDataFile, useANSClassFile); return FileExist(YalmDataFile); } bool ANYLOBBTRE::CreateTrainingEngine(std::string projectName, int experimentNumber, int extractorType, long numberStep, int batchSize, double learningRate)// extractorType =0: FAST, 1: ACCURATE; 2: VERY ACCURATE { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string experimentParentFolder = GetParentFolder(experimentPath); std::string evaluationModelFile = experimentParentFolder + "\\yolo11n.pt"; if (!FolderExist(experimentParentFolder))fs::create_directory(experimentParentFolder); if (!FileExist(evaluationModelFile)) CopyFile(yolo11nModelFile, evaluationModelFile); //0. Check if the experiement folder exist std::string zipExperimentPath = experimentPath + ".zip"; if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath); if (!unzipResult) { unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath); } if (unzipResult) { DeleteFile(zipExperimentPath); } DeleteWeightsFiles(experimentPath); } //1. Create a brand new experiment folder if (!FolderExist(experimentPath))fs::create_directory(experimentPath); //2. Delete the old configuration files std::string trainCfgFile = experimentPath + "\\train.cfg"; if (FileExist(trainCfgFile))DeleteFile(trainCfgFile); //3. Copy classes.names file to the experiment folder std::string classesNamesFile = projectPath + "\\data\\classes.names"; std::string experimentClassesNamesFile = experimentPath + "\\classes.names"; if (FileExist(classesNamesFile))CopyFile(classesNamesFile, experimentClassesNamesFile); //4. Generate the configuration file std::ofstream outFile(trainCfgFile); outFile << "project_name=" << projectName << "\n"; outFile << "experiment_number=" << experimentNumber << "\n"; outFile << "extractor_type=" << extractorType << "\n"; outFile << "batch=" << batchSize << "\n"; outFile << "epochs=" << numberStep << "\n"; outFile << "lr0=" << learningRate << "\n"; outFile.close(); return true; } std::map ANYLOBBTRE::ExtractDataFromCfgFile(const std::string& filePath) { std::map data; data.clear(); if (FileExist(filePath)) { std::ifstream file(filePath); if (!file.is_open()) { return data; } std::string line; while (std::getline(file, line)) { std::istringstream iss(line); std::string key, value; if (std::getline(iss, key, '=') && std::getline(iss, value)) { data[key] = value; } } } return data; } std::string ANYLOBBTRE::GenerateTrainingCommand(std::string projectName, int experimentNumber, int extractorType) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string trainScript = _engineDirectory + "\\yolo.exe"; std::string trainCfgFile = experimentPath + "\\train.cfg"; std::string projectDataPath = projectPath + "\\data\\data.yaml"; std::string preTrainWeightFile = experimentPath + "\\train_last.pt"; // Override the preTrainWeightFile if the experiment folder is already zipped std::string zipExperimentPath = experimentPath + ".zip"; std::string experimentParentFolder = GetParentFolder(experimentPath); if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath); if (!unzipResult) { unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath); } if (unzipResult) { DeleteFile(zipExperimentPath); } } //Copy classes.names file to the experiment folder std::string classesNamesFile = projectPath + "\\data\\classes.names"; std::string classNamesFile = experimentPath + "\\classes.names"; if (FileExist(classNamesFile))DeleteFile(classNamesFile); if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile); std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath); // Check if the last train weight file is exist std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName; std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt"; if (FileExist(lastTrainWeightFile)) { CopyFile(lastTrainWeightFile, preTrainWeightFile); DeleteFolder(experimentResultPath); } if (!FileExist(preTrainWeightFile)) { switch (extractorType) {// then use templae case 0: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8n.pt"; break; case 1: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8m.pt"; break; case 2: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8x.pt"; break; default: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8n.pt"; break; } } if (!FileExist(trainScript)) this->_logger.LogError("ANYLOBBTRE::GenerateTrainingCommand", "ANSODTRE Engine is not installed", __FILE__, __LINE__); if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLOBBTRE::GenerateTrainingCommand", "Missing train.cfg file", __FILE__, __LINE__); if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLOBBTRE::GenerateTrainingCommand", "Missing pre-trained weight file", __FILE__, __LINE__); if (!FileExist(projectDataPath)) this->_logger.LogError("ANYLOBBTRE::GenerateTrainingCommand", "Missing data yalm file", __FILE__, __LINE__); if ((FileExist(trainScript)) && (FileExist(trainCfgFile)) && (FileExist(preTrainWeightFile)) && (FileExist(projectDataPath))) { // Extract information from train.cfg file std::string epochs = "2000"; std::string learningRate = "0.01"; std::string batch = "-1"; // auto batch size std::map cfgMapData = ExtractDataFromCfgFile(trainCfgFile); if (cfgMapData.size() > 0) { epochs = cfgMapData.at("epochs"); learningRate = cfgMapData.at("lr0"); batch = cfgMapData.at("batch"); } std::string trainCommand = trainScript + " obb train model=" + "\"" + preTrainWeightFile + "\"" + " data=" + "\"" + projectDataPath + "\"" + " imgsz=640" + " workers=8" + " batch=" + batch + " device=0" + " epochs=" + epochs + " patience=0" + " plots=False" + " name=" + _projectName + " project=" + "\"" + experimentPath + "\""; return trainCommand; } this->_logger.LogError("ANYLOBBTRE::GenerateTrainingCommand", "Missing configuration files", __FILE__, __LINE__); return ""; } else { this->_logger.LogError("ANYLOBBTRE::GenerateTrainingCommand", "project path is not exist", __FILE__, __LINE__); return ""; } } std::string ANYLOBBTRE::GenerateCustomTrainingCommand(std::string projectName, int experimentNumber, int extractorType, std::string pretrainedModel) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string trainScript = _engineDirectory + "\\yolo.exe"; std::string trainCfgFile = experimentPath + "\\train.cfg"; std::string projectDataPath = projectPath + "\\data\\data.yaml"; std::string preTrainWeightFile = pretrainedModel; if (!FileExist(preTrainWeightFile)) preTrainWeightFile = experimentPath + "\\train_last.pt"; // Override the preTrainWeightFile if the experiment folder is already zipped std::string zipExperimentPath = experimentPath + ".zip"; std::string experimentParentFolder = GetParentFolder(experimentPath); if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath); if (!unzipResult) { unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath); } if (unzipResult) { DeleteFile(zipExperimentPath); } } //Copy classes.names file to the experiment folder std::string classesNamesFile = projectPath + "\\data\\classes.names"; std::string classNamesFile = experimentPath + "\\classes.names"; if (FileExist(classNamesFile))DeleteFile(classNamesFile); if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile); std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath); // Check if the last train weight file is exist std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName; std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt"; if (FileExist(lastTrainWeightFile)) { CopyFile(lastTrainWeightFile, preTrainWeightFile); DeleteFolder(experimentResultPath); } if (!FileExist(preTrainWeightFile)) { switch (extractorType) {// then use templae case 0: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8n.pt"; break; case 1: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8m.pt"; break; case 2: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8x.pt"; break; default: preTrainWeightFile = _modelTemplateDirectory + "\\ob\\ob8n.pt"; break; } } if (!FileExist(trainScript)) this->_logger.LogError("ANYLOBBTRE::GenerateCustomTrainingCommand", "ANSODTRE Engine is not installed", __FILE__, __LINE__); if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLOBBTRE::GenerateCustomTrainingCommand", "Missing train.cfg file", __FILE__, __LINE__); if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLOBBTRE::GenerateCustomTrainingCommand", "Missing pre-trained weight file", __FILE__, __LINE__); if (!FileExist(projectDataPath)) this->_logger.LogError("ANYLOBBTRE::GenerateCustomTrainingCommand", "Missing data yalm file", __FILE__, __LINE__); if ((FileExist(trainScript)) && (FileExist(trainCfgFile)) && (FileExist(preTrainWeightFile)) && (FileExist(projectDataPath))) { // Extract information from train.cfg file std::string epochs = "2000"; std::string learningRate = "0.01"; std::string batch = "-1"; // auto batch size std::map cfgMapData = ExtractDataFromCfgFile(trainCfgFile); if (cfgMapData.size() > 0) { epochs = cfgMapData.at("epochs"); learningRate = cfgMapData.at("lr0"); batch = cfgMapData.at("batch"); } std::string trainCommand = trainScript + " obb train model=" + "\"" + preTrainWeightFile + "\"" + " data=" + "\"" + projectDataPath + "\"" + " imgsz=640" + " workers=8" + " batch=" + batch + " device=0" + " epochs=" + epochs + " patience=0" + " plots=False" + " name=" + _projectName + " project=" + "\"" + experimentPath + "\""; return trainCommand; } this->_logger.LogError("ANYLOBBTRE::GenerateCustomTrainingCommand", "Missing configuration files", __FILE__, __LINE__); return ""; } else { this->_logger.LogError("ANYLOBBTRE::GenerateCustomTrainingCommand", "project path is not exist", __FILE__, __LINE__); return ""; } } std::string ANYLOBBTRE::GenerateEvaluateModelCommand(std::string projectName, int experimentNumber, bool& zipModelExist) { _projectName = Space2Underscore(projectName); zipModelExist = false; std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string trainScript = _engineDirectory + "\\yolo.exe"; std::string trainCfgFile = experimentPath + "\\train.cfg"; std::string preTrainWeightFile = experimentPath + "\\train_last.pt"; std::string projectDataPath = projectPath + "\\data\\data.yaml"; // Override the preTrainWeightFile if the experiment folder is already zipped std::string zipExperimentPath = experimentPath + ".zip"; std::string experimentParentFolder = GetParentFolder(experimentPath); if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath); if (!unzipResult) { unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath); } if (unzipResult) { zipModelExist = true; } } //Copy classes.names file to the experiment folder std::string classesNamesFile = projectPath + "\\data\\classes.names"; std::string classNamesFile = experimentPath + "\\classes.names"; if (FileExist(classNamesFile))DeleteFile(classNamesFile); if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile); //// Check if the last train weight file is exist //std::string experimentResultPath = experimentPath + "\\" + _projectName; //std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt"; std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath); // Check if the last train weight file is exist std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName; std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt"; if (FileExist(lastTrainWeightFile)) { CopyFile(lastTrainWeightFile, preTrainWeightFile); DeleteFolder(experimentResultPath); } if (!FileExist(trainScript)) this->_logger.LogError("ANYLOBBTRE::GenerateEvaluateModelCommand", "ANYLOBBTRE Engine is not installed", __FILE__, __LINE__); if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLOBBTRE::GenerateEvaluateModelCommand", "Missing train.cfg file", __FILE__, __LINE__); if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLOBBTRE::GenerateEvaluateModelCommand", "Missing pre-trained weight file", __FILE__, __LINE__); if (!FileExist(classNamesFile)) this->_logger.LogError("ANYLOBBTRE::GenerateEvaluateModelCommand", "Missing classes.names file", __FILE__, __LINE__); if ((FileExist(trainScript)) && (FileExist(trainCfgFile)) && (FileExist(preTrainWeightFile)) && (FileExist(classNamesFile))) { std::string evaluateCommand = trainScript + " detect val model=" + "\"" + preTrainWeightFile + "\"" + " data=" + "\"" + projectDataPath + "\"" + " split=val" + " imgsz=640" + " workers=8" + " batch=8" + " device=0" + " conf=0.25" + " plots=False" + " save_json=False" + " iou=0.6"; return evaluateCommand; } this->_logger.LogError("ANYLOBBTRE::GenerateEvaluateModelCommand", "Missing configuration files", __FILE__, __LINE__); return ""; } else { this->_logger.LogError("ANYLOBBTRE::GenerateEvaluateModelCommand", "project path is not exist", __FILE__, __LINE__); return ""; } } bool ANYLOBBTRE::DownloadModel(std::string projectName, int experimentNumber, std::string destZipDirectory, int modelMode) { _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { //0. Get all files in the experiment folder std::string trainScript = _engineDirectory + "\\yolo.exe"; std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string zipExperimentPath = experimentPath + ".zip"; std::string trainCfgFile = experimentPath + "\\train.cfg"; int extractorType = 0; if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it if (ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath)) { DeleteFile(zipExperimentPath); } else { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "Can not unzip model zip file to extract information. Invalid password.", __FILE__, __LINE__); return false; } } //Copy classes.names file to the experiment folder std::string classesNamesFile = projectPath + "\\data\\classes.names"; std::string classNamesFile = experimentPath + "\\classes.names"; if (FileExist(classNamesFile))DeleteFile(classNamesFile); if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile); // Extract information from train.cfg file std::map cfgMapData = ExtractDataFromCfgFile(trainCfgFile); if (cfgMapData.size() > 0) { extractorType = std::stoi(cfgMapData.at("extractor_type")); } std::string exportDirectory = projectPath + "\\exports"; if (!FolderExist(exportDirectory))fs::create_directory(exportDirectory); std::string exportExperimentFolder = exportDirectory + "\\" + projectName + "_" + std::to_string(experimentNumber); std::string exportExperimentZip = exportExperimentFolder + ".zip"; if (FileExist(exportExperimentZip))DeleteFile(exportExperimentZip); if (!FolderExist(exportExperimentFolder))fs::create_directory(exportExperimentFolder); //// Check if the last train weight file is exist std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath); // Check if the last train weight file is exist std::string preTrainWeightFile = experimentPath + "\\train_last.pt"; std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName; std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt"; if (FileExist(lastTrainWeightFile)) { if (FileExist(preTrainWeightFile))DeleteFile(preTrainWeightFile); CopyFile(lastTrainWeightFile, preTrainWeightFile); DeleteFolder(experimentResultPath); } if (!FileExist(preTrainWeightFile)) { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "Missing pre-trained weight file", __FILE__, __LINE__); return false; } // 1. Get the names of classes.names std::ifstream inFile(classNamesFile); if (!inFile.is_open()) { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "cannot open file to save model information.", __FILE__, __LINE__); return false; } std::vector classesNames; std::string line; while (std::getline(inFile, line)) { classesNames.push_back(line); } inFile.close(); // 2. Copy the classes.names file to the export folder std::string exportClassesNameFile = exportExperimentFolder + "\\classes.names"; CopyFile(classNamesFile, exportClassesNameFile); // 3. Create Categories folder std::string categoriesFolder = exportExperimentFolder + "\\Categories"; if (FolderExist(categoriesFolder))DeleteFolder(categoriesFolder); fs::create_directory(categoriesFolder); for (size_t i = 0; i < classesNames.size(); ++i) { std::ofstream outFile(categoriesFolder + "/" + classesNames[i] + "." + std::to_string(i + 1)); outFile.close(); } //4. Create a text file name yolo.txt if (modelMode == 0) { // This is the tensorRT onnx model std::string yoloTxtFile = yoloTxtFile = exportExperimentFolder + "\\tensorrtseg.txt"; std::ofstream outFile(yoloTxtFile); outFile.close(); std::string exportCommand = trainScript + " export model=" + preTrainWeightFile + " format=onnx" + " dynamic=True " + " opset=12" + " nms=True" + " simplify=False"; if (!FileExist(preTrainWeightFile)) { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "Pretrain file does not exist.", __FILE__, __LINE__); return false; } std::string onnxFile = experimentPath + "\\train_last.onnx"; if (FileExist(onnxFile))DeleteFile(onnxFile); ExecuteCommand(exportCommand.c_str()); if (!FileExist(onnxFile)) { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "cannot generate onnx file.", __FILE__, __LINE__); return false; } // Copy the onnx file to the export folder std::string exportOnnxFile = exportExperimentFolder + "\\train_last.onnx"; CopyFile(onnxFile, exportOnnxFile); DeleteFile(onnxFile); } else { std::string yoloTxtFile = exportExperimentFolder + "\\openvinoseg.txt"; std::ofstream outFile(yoloTxtFile); outFile.close(); std::string pythonDir = GetParentFolder(_engineDirectory); std::string pythonScript = pythonDir + "\\python.exe"; std::string modelOptimiser = _modelTemplateDirectory + "\\ModelOptimizer.py"; std::string exportCommand = pythonScript + " " + modelOptimiser + " " + preTrainWeightFile; if (!_isLatestEngine) exportCommand = trainScript + " export model=" + preTrainWeightFile + " half = False" + " format=openvino" + " nms=True" + " simplify=False"; if (!FileExist(preTrainWeightFile)) { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "Pretrain file does not exist.", __FILE__, __LINE__); return false; } ExecuteCommand(exportCommand.c_str()); std::string xmlTrainedModelFile = exportExperimentFolder + "\\train_last.xml"; std::string binTrainedModelFile = exportExperimentFolder + "\\train_last.bin"; if (FileExist(xmlTrainedModelFile)) DeleteFile(xmlTrainedModelFile); if (FileExist(binTrainedModelFile)) DeleteFile(binTrainedModelFile); std::string outputExportedFolder = experimentPath + "\\train_last_openvino_model"; std::string xmlFile = outputExportedFolder + "\\train_last.xml"; std::string binFile = outputExportedFolder + "\\train_last.bin"; if (!FileExist(xmlFile)) { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "cannot generate openvino file.", __FILE__, __LINE__); return false; } CopyFile(xmlFile, xmlTrainedModelFile); CopyFile(binFile, binTrainedModelFile); DeleteFolder(outputExportedFolder); } //6. Zip the experiment folder if (ZipFolderWithPassword(experimentPath.c_str(), zipExperimentPath.c_str(), _zipPassword.c_str())) { DeleteFolder(experimentPath); } // 7. Zip the Experiment folder if (ZipFolderWithPassword(exportExperimentFolder.c_str(), exportExperimentZip.c_str(), _zipPassword.c_str())) { DeleteFolder(exportExperimentFolder); std::string destZipFilePath = destZipDirectory + "\\ANS_" + projectName + "(GPU)_" + std::to_string(experimentNumber) + ".zip"; if (modelMode > 0)destZipFilePath = destZipDirectory + "\\ANS_" + projectName + "(CPU)_" + std::to_string(experimentNumber) + ".zip"; //std::string destZipFilePath = destZipDirectory + "\\" + projectName + "_" + std::to_string(experimentNumber) + ".zip"; CopyFile(exportExperimentZip, destZipFilePath); if (FileExist(destZipFilePath)) { //DeleteFile(exportExperimentZip); return true; } return false; } this->_logger.LogError("ANYLOBBTRE::DownloadModel", "Can not unzip model zip file to extract information. Invalid password.", __FILE__, __LINE__); return false; } else { this->_logger.LogError("ANYLOBBTRE::DownloadModel", "Invalid project path.", __FILE__, __LINE__); return false; } } std::string ANYLOBBTRE::EvaluateModel(std::string projectName, int experimentNumber) { bool zipModelExist = false; _projectName = Space2Underscore(projectName); std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string zipExperimentPath = experimentPath + ".zip"; std::string preTrainWeightFile = experimentPath + "\\train_last.pt"; std::string evaluateCommand = GenerateEvaluateModelCommand(_projectName, experimentNumber, zipModelExist); if (evaluateCommand.empty()) { this->_logger.LogError("ANSODTRE::EvaluateModel", "evaluateCommand is empty", __FILE__, __LINE__); return ""; } if (!FileExist(preTrainWeightFile)) { this->_logger.LogError("ANSODTRE::EvaluateModel", "pre-trained model is not exist", __FILE__, __LINE__); return ""; } return evaluateCommand; } else { this->_logger.LogError("ANYLOBBTRE::EvaluateModel", "Invalid project path.", __FILE__, __LINE__); return ""; } } std::string ANYLOBBTRE::ParseTrainingResults(std::string trainingResults) { if (trainingResults.empty()) return "step: 0; loss=0.00; map=0.00"; try { // Regex pattern to extract the first integer from "1/2000" std::regex epoch_regex(R"((\d+)/\d+)"); std::smatch epoch_match; std::string epoch = "0"; std::string loss = "0.00"; std::string map = "0.00"; if (std::regex_search(trainingResults, epoch_match, epoch_regex)) { epoch = epoch_match[1].str(); } // Regex to find the line containing "all" and then extract the last two items (0.994 and 0.81) from that line std::regex line_regex(R"(all.*?(\d+\.\d+)\s+(\d+\.\d+)$)"); std::smatch line_match; if (std::regex_search(trainingResults, line_match, line_regex)) { loss = line_match[1].str(); map = line_match[2].str(); } std::ostringstream status; status << "step: " << epoch << "; loss=" << loss << "; map=" << map; std::string result = status.str(); std::cout << "result:" << result << std::endl; return result; } catch (std::exception& e) { this->_logger.LogFatal("ANYLOBBTRE::ParseTrainingResults", e.what(), __FILE__, __LINE__); return "step: 0; loss=0.00; map=0.00"; } } bool ANYLOBBTRE::CheckEngineStatus() { const char* targetProcess = "yolo.exe"; // Replace with the target process name if (IsProcessRunning(targetProcess)) { return true; } else { return false; } } bool ANYLOBBTRE::CheckEngine() { std::string inferenceCommand = "C:\\ProgramData\\ANSCENTER\\Python311\\Scripts\\yolo.exe detect predict model = \"C:\\ProgramData\\ANSCENTER\\Python311\\Lib\\site-packages\\data\\od\\od8m.pt\" source = \"C:\\ProgramData\\ANSCENTER\\Shared\\test.jpg\" save = False"; std::string result = ExecuteCommand(inferenceCommand.c_str()); if (result.empty()) { return false; } else { std::string target = "640x480 4 persons, 1 bus"; // Check if the target text is found in the input text if (result.find(target) != std::string::npos) { std::cout << "Engine is installed correctly." << std::endl; return true; // Text is found } else { std::cout << "Engine is not installed correctly." << std::endl; return false; // Text is not found } } } //Private functions std::string ANYLOBBTRE::ParseMAP(const std::string& text) { std::regex valuePattern("all\\s+\\d+\\s+\\d+\\s+\\d\\.\\d+\\s+\\d\\.\\d+\\s+\\d\\.\\d+\\s+(\\d\\.\\d+)"); std::smatch matches; if (std::regex_search(text, matches, valuePattern) && matches.size() > 1) { return matches[1].str(); } else { return "0.00"; } } std::string ANYLOBBTRE::AdjustRootDirectory(const std::string& root) { std::filesystem::path rootPath(root); std::vector subdirectories; for (const auto& entry : std::filesystem::directory_iterator(rootPath)) { if (entry.is_directory()) { subdirectories.push_back(entry.path()); } } if (subdirectories.size() == 1) { return subdirectories[0].string(); } else { return root; } } std::string ANYLOBBTRE::GenerateClassNames(std::string labelMapFile) { std::ifstream inFile(labelMapFile); if (!inFile.is_open()) { return "[]"; } std::string classesNames = "["; std::string line; // Updated pattern: allows optional spaces between name and the colon std::regex namePattern(R"(name\s*:\s*'(.*)')"); int fileLine = 0; while (std::getline(inFile, line)) { std::smatch matches; std::cout << line << std::endl; if (std::regex_search(line, matches, namePattern) && matches.size() > 1) { classesNames += matches[1].str() + ","; fileLine++; } } inFile.close(); // Remove the last comma if there is one if (classesNames.back() == ',') { classesNames.pop_back(); } classesNames += "]"; return classesNames; } // -data.yaml // path: sourceDataPath // train: images\train // val : images\val // names : [plate] bool ANYLOBBTRE::GenerateObjectDataFile(std::string sourcePbDataPath, std::string objectDataFile, bool useANSClassFile) { if (!FileExist(sourcePbDataPath)) return false; std::string projectDataName = GetParentFolder(objectDataFile); if (FileExist(objectDataFile))DeleteFile(objectDataFile); if (useANSClassFile) { // Get the class names from the ANSClass.json file std::vector classVec = ParseANSClass(sourcePbDataPath); std::string classNames= ConvertVectorToClass(classVec); std::ofstream outFile(objectDataFile); outFile << "path: " << projectDataName << "\n"; outFile << "train: " << "images\\train" << "\n"; outFile << "val: " << "images\\val" << "\n"; outFile << "names: " << classNames << "\n"; outFile.close(); } else { // Get the class names from the pbtxt file std::string classNames = GenerateClassNames(sourcePbDataPath); std::ofstream outFile(objectDataFile); outFile << "path: " << projectDataName << "\n"; outFile << "train: " << "images\\train" << "\n"; outFile << "val: " << "images\\val" << "\n"; outFile << "names: " << classNames << "\n"; outFile.close(); } return true; } void ANYLOBBTRE::GenerateTrainDataSet(const std::string& sourceFolder, const std::string& imageFolder, const std::string& labelFolder, double trainRatio) { // 1. Create image and label folders if (FolderExist(imageFolder))DeleteFolder(imageFolder); if (FolderExist(labelFolder))DeleteFolder(labelFolder); fs::create_directory(imageFolder); fs::create_directory(labelFolder); // 2. Create train and val folders std::string trainImagesFolder = imageFolder + "\\train"; std::string valImagesFolder = imageFolder + "\\val"; if (FolderExist(trainImagesFolder))DeleteFolder(trainImagesFolder); if (FolderExist(valImagesFolder))DeleteFolder(valImagesFolder); fs::create_directory(trainImagesFolder); fs::create_directory(valImagesFolder); std::string trainLabelsFolder = labelFolder + "\\train"; std::string valLabelsFolder = labelFolder + "\\val"; if (FolderExist(trainLabelsFolder))DeleteFolder(trainLabelsFolder); if (FolderExist(valLabelsFolder))DeleteFolder(valLabelsFolder); fs::create_directory(trainLabelsFolder); fs::create_directory(valLabelsFolder); std::unordered_map> fileGroups; for (const auto& entry : std::filesystem::directory_iterator(sourceFolder)) { std::string extension = entry.path().extension().string(); boost::algorithm::to_lower(extension); if (entry.is_regular_file()) { if (extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".bmp") fileGroups[entry.path().stem().string()].push_back(entry.path()); } } std::vector groupNames; for (const auto& group : fileGroups) { groupNames.push_back(group.first); } size_t trainSize = static_cast(groupNames.size() * trainRatio); for (size_t i = 0; i < groupNames.size(); ++i) { const auto& groupName = groupNames[i]; const auto& files = fileGroups[groupName]; for (const auto& file : files) { std::string txtSourceFile = sourceFolder + "\\" + GetFileNameWithoutExtension(file.filename().string()) + ".txt"; if (i < trainSize) { std::string fileName = file.filename().string(); std::string fileExtension = GetFileExtension(fileName); std::string txtTrainFile = trainLabelsFolder + "\\" + GetFileNameWithoutExtension(file.filename().string()) + ".txt"; if (fileExtension == ".txt" || fileExtension == ".jpeg" || fileExtension == ".jpg" || fileExtension == ".png" || fileExtension == ".bmp") { if (FileExist(txtSourceFile)) { std::filesystem::copy(file, trainImagesFolder / file.filename()); CopyFile(txtSourceFile, txtTrainFile); } } } else { std::string fileName = file.filename().string(); std::string fileExtension = GetFileExtension(fileName); std::string txtTestFile = valLabelsFolder + "\\" + GetFileNameWithoutExtension(file.filename().string()) + ".txt"; if (fileExtension == ".txt" || fileExtension == ".jpeg" || fileExtension == ".jpg" || fileExtension == ".png" || fileExtension == ".bmp") { if (FileExist(txtSourceFile)) { std::filesystem::copy(file, valImagesFolder / file.filename()); CopyFile(txtSourceFile, txtTestFile); } } } } } } bool ANYLOBBTRE::GenerateConfigFile(const std::string& templateFile, const std::string& outputFile, const std::map& replacements) { std::ifstream inFile(templateFile); std::ofstream outFile(outputFile); if (!inFile.is_open() || !outFile.is_open()) { return false; } std::string line; while (std::getline(inFile, line)) { for (const auto& replacement : replacements) { size_t pos = line.find("$" + replacement.first); while (pos != std::string::npos) { line.replace(pos, replacement.first.size() + 1, replacement.second); pos = line.find("$" + replacement.first, pos + replacement.second.size()); } } outFile << line << "\n"; } inFile.close(); outFile.close(); return FileExist(outputFile); } int ANYLOBBTRE::ReadClassesValue(const std::string& filePath) { std::ifstream inFile(filePath); if (!inFile.is_open()) { return 0; } std::ofstream outFile(filePath, std::ios_base::app); if (!outFile.is_open()) { return 0; } std::string currentFolderName = GetParentFolder(filePath); outFile << "backup = " << currentFolderName << "\n"; outFile.close(); std::string line; while (std::getline(inFile, line)) { if (line.find("classes") == 0) { size_t pos = line.find("="); if (pos != std::string::npos) { std::string classesValue = line.substr(pos + 1); inFile.close(); return std::stoi(classesValue); } } } inFile.close(); return 0; } void ANYLOBBTRE::DeleteWeightsFiles(const std::string& folderPath) { for (const auto& entry : std::filesystem::directory_iterator(folderPath)) { if (entry.is_regular_file() && entry.path().extension() == ".weights" && entry.path().filename() != "train_last.weights") { std::filesystem::remove(entry.path()); } } } std::string ANYLOBBTRE::GenerateTestCommand(std::string projectName, int experimentNumber, bool& zipModelExist, const std::string& testDataFolder) { _projectName = Space2Underscore(projectName); zipModelExist = false; std::string projectPath = FindFilePathInFileList(_projects, _projectName); if (FolderExist(projectPath)) { std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber); std::string trainScript = _engineDirectory + "\\yolo.exe"; std::string trainCfgFile = experimentPath + "\\train.cfg"; std::string preTrainWeightFile = experimentPath + "\\train_last.pt"; std::string projectDataPath = projectPath + "\\data\\data.yaml"; // Override the preTrainWeightFile if the experiment folder is already zipped std::string zipExperimentPath = experimentPath + ".zip"; std::string experimentParentFolder = GetParentFolder(experimentPath); if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath); if (!unzipResult) { unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath); } if (unzipResult) { zipModelExist = true; } } //Copy classes.names file to the experiment folder std::string classesNamesFile = projectPath + "\\data\\classes.names"; std::string classNamesFile = experimentPath + "\\classes.names"; if (FileExist(classNamesFile))DeleteFile(classNamesFile); if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile); std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath); // Check if the last train weight file is exist std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName; std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt"; if (FileExist(lastTrainWeightFile)) { CopyFile(lastTrainWeightFile, preTrainWeightFile); DeleteFolder(experimentResultPath); } if (!FileExist(trainScript)) this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "ANYLOBBTRE Engine is not installed", __FILE__, __LINE__); if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "Missing train.cfg file", __FILE__, __LINE__); if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "Missing pre-trained weight file", __FILE__, __LINE__); if (!FileExist(classNamesFile)) this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "Missing classes.names file", __FILE__, __LINE__); // Create test data yaml file std::string testYamlPath = testDataFolder + "\\data.yaml"; std::string testDataFolder = testDataFolder + "\\data"; // Content of the test data yaml file // path: testDataFolder // val : images // names : [Cavity,Fillings,Impacted Tooth,Implant] // Classes.names content // Cavity // Fillings // Impacted Tooth // Implant std::string classNames; classNames = "["; // read line by line from the classNamesFile std::ifstream inFile(classNamesFile); if (!inFile.is_open()) { this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "cannot open file to save model information.", __FILE__, __LINE__); return ""; } std::string line; while (std::getline(inFile, line)) { classNames += line + ","; } inFile.close(); // Remove the last comma if there is one if (classNames.back() == ',') { classNames.pop_back(); } classNames += "]"; std::ofstream outFile(testYamlPath); outFile << "path: " << testDataFolder << "\n"; outFile << "val: " << "images" << "\n"; outFile << "names: " << classNames << "\n"; outFile.close(); if ((FileExist(trainScript)) && (FileExist(trainCfgFile)) && (FileExist(preTrainWeightFile)) && (FileExist(classNamesFile))) { std::string evaluateCommand = trainScript + " detect val model=" + "\"" + preTrainWeightFile + "\"" + " data=" + "\"" + testYamlPath + "\"" + " split=val" + " imgsz=640" + " workers=8" + " batch=8" + " device=0" + " conf=0.25" + " plots=False" + " save_json=False" + " iou=0.6"; return evaluateCommand; } this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "Missing configuration files", __FILE__, __LINE__); return ""; } else { this->_logger.LogError("ANYLOBBTRE::GenerateTestCommand", "project path is not exist", __FILE__, __LINE__); return ""; } } }