ANSCENTER/ANSCORE
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c9ea3ac5a2e3d17853b536dfef8991756f65ac7d
ANSCORE/ANSODTrainingEngine/ANSYOLOCLTrainingEngine.cpp

1146 lines
46 KiB
C++
Raw Normal View History

Initial setup for CLion
2026-03-28 16:54:11 +11:00
#include "ANSYOLOCLTrainingEngine.h"
#include "Utility.h"
#include <random>
#include <cstdio>
#include <memory>
#include <regex>
namespace ANSCENTER
{
ANYLCLTRE::ANYLCLTRE() {
_projectDirectory = "C:\\ProgramData\\Sh7O7nUe7vJ\\anscl";
_engineDirectory = "C:\\ProgramData\\ANSCENTER\\Python\\Scripts";
_modelTemplateDirectory = "C:\\ProgramData\\ANSCENTER\\Shared";
_currentExperimentNumber = 1;
_isLicenseValid = false;
_licenseKey = "";
_zipPassword = "";
_changeProjectDirectory = false;
_isLatestEngine = true;// Default is YoloV10 engine
}
ANYLCLTRE::~ANYLCLTRE() {
_projects.clear();
_zipPassword = "";
_isLicenseValid = false;
_changeProjectDirectory = false;
_isLatestEngine = true;
}
void ANYLCLTRE::CheckLicense() {
try {
_isLicenseValid = ANSCENTER::ANSLicenseHelper::LicenseVerification(_licenseKey, 1008, "ANSTS");//Default productId=1008 (ANSTS)
}
catch (std::exception& e) {
this->_logger.LogFatal("ANYLCLTRE::CheckLicense()", e.what(), __FILE__, __LINE__);
}
}
bool ANYLCLTRE::Init(std::string licenseKey,
std::string projectDirectory,
std::string engineDirectory,
std::string modelTemplateDirectory,
std::string modelZipPassword,
bool isLatestEngine)
{
_licenseKey = licenseKey;
_isLatestEngine = isLatestEngine;
CheckLicense();
if (!_isLicenseValid) {
this->_logger.LogError("ANYLCLTRE::Init", "Invalid license key", __FILE__, __LINE__);
return false;
}
if (projectDirectory.empty())_projectDirectory = "C:\\ProgramData\\Sh7O7nUe7vJ\\anscl";
else _projectDirectory = projectDirectory + "\\anscl";
_engineDirectory = engineDirectory;
_modelTemplateDirectory = modelTemplateDirectory;
if (_engineDirectory.empty())_engineDirectory = "C:\\ProgramData\\ANSCENTER\\Python311\\Scripts";
if (_modelTemplateDirectory.empty())_modelTemplateDirectory = "C:\\ProgramData\\ANSCENTER\\Python311\\Lib\\site-packages\\data";
// Change the project directory to 1 level down if there is 1 folder in the project directory
if (!FolderExist(_projectDirectory)) fs::create_directory(_projectDirectory);
_modelZipPassword = modelZipPassword;
_zipPassword = "Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==";
_modelZipPassword = _zipPassword;
// Some logging in here
std::string _ansdnnEngine = _engineDirectory + "\\yolo.exe";
if (!FileExist(_ansdnnEngine))this->_logger.LogError("ANYLCLTRE::Init", "ANSDL Engine is not installed. Please install ANS Runtime Engine", __FILE__, __LINE__);
std::string fastWeight = _modelTemplateDirectory + "\\cl\\cl8n.pt";
std::string accurateWeight = _modelTemplateDirectory + "\\cl\\cl8m.pt";
std::string veryAccurateWeight = _modelTemplateDirectory + "\\cl\\cl8x.pt";
if (!_isLatestEngine) {// Yolov8
fastWeight = _modelTemplateDirectory + "\\cl\\cl8n.pt";
accurateWeight = _modelTemplateDirectory + "\\cl\\cl8m.pt";
veryAccurateWeight = _modelTemplateDirectory + "\\cl\\cl8x.pt";
}
yolo11nModelFile = _modelTemplateDirectory + "\\yolo11n.pt";
if (!FileExist(fastWeight))this->_logger.LogError("ANYLCLTRE::Init", "FAST model file is missing", __FILE__, __LINE__);
if (!FileExist(accurateWeight))this->_logger.LogError("ANYLCLTRE::Init", "ACCURATE model file is missing", __FILE__, __LINE__);
if (!FileExist(veryAccurateWeight))this->_logger.LogError("ANYLCLTRE::Init", "VERYACCURATE model file is missing", __FILE__, __LINE__);
if (!FileExist(yolo11nModelFile))this->_logger.LogError("ANYLCLTRE::Init", "Evaluate model file is missing", __FILE__, __LINE__);
if (FolderExist(_projectDirectory) &&
FolderExist(_engineDirectory) &&
FolderExist(_modelTemplateDirectory))return true;
return false;
}
std::vector<std::string> ANYLCLTRE::GetProjects() {
_projects.clear();
std::vector<std::string> _projectNames;
for (auto& entry : std::filesystem::directory_iterator(_projectDirectory))
{
if (entry.is_directory())
{
_projects.push_back(entry.path().string());
_projectNames.push_back(ExtractFolderName(entry.path().string()));
}
}
return _projectNames;
}
std::vector<std::string> ANYLCLTRE::GetProjectExperiments(std::string projectName) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
std::string experimentPath = projectPath + "\\experiments";
if (FolderExist(experimentPath)) {
std::vector<std::string> files;
std::vector<std::string> experiments;
for (const auto& entry : std::filesystem::directory_iterator(experimentPath))
{
if (entry.is_regular_file() && entry.path().extension() == ".zip")
{
experiments.push_back(GetFileNameWithoutExtension(entry.path().string()));
}
}
return experiments;
}
else {
std::vector<std::string> experiments;
return experiments;
}
}
std::string ANYLCLTRE::GetProjectExperimentStatus(std::string projectName, int experimentNumber) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
std::string experimentFolder = projectPath + "\\experiments";
if (FolderExist(experimentFolder)) {
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber) + ".zip";
if (FileExist(experimentPath)) {
return "TRAINED";
}
else
{
std::string configurationPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber) + "\\train.cfg";
if (FileExist(configurationPath))
{
std::string tempWeightFile = projectPath + "\\experiments\\" + std::to_string(experimentNumber) + "\\" + _projectName + "\\weights\\best.pt";
if (FileExist(tempWeightFile)) return "RUNNING";
else return "CONFIGURED";
}
else return "UPLOADED";
}
}
else {
std::string dataPath = projectPath + "\\data";
if (FileExist(dataPath))
{
return "CREATED";
}
else {
return "UPLOADED";
}
}
}
else return "NOTFOUND";
}
bool ANYLCLTRE::CreateProject(std::string projectName) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (projectPath.empty()) {
std::string newProjectPath = _projectDirectory + "\\" + _projectName;
if (FolderExist(newProjectPath))return false;
fs::create_directory(newProjectPath);
_projects.push_back(newProjectPath);
return true;
}
return false;
}
bool ANYLCLTRE::DeleteProject(std::string projectName) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
return DeleteFolder(projectPath);
}
return false;
}
bool ANYLCLTRE::SetWorkingDirectory(std::string workingDirectory) {
if (FolderExist(workingDirectory)) {
size_t lastSlashPos = workingDirectory.find_last_of("\\/");
if (lastSlashPos != std::string::npos && workingDirectory.substr(lastSlashPos + 1) == "data") {
workingDirectory = workingDirectory.substr(0, lastSlashPos);
}
_workingDirectory = workingDirectory;
std::string dataPath = _workingDirectory + "\\data"; // contain folders as categories
std::string pbLabelPath = dataPath + "\\CatergoryNames.txt"; // Contain class names
if (FileExist(pbLabelPath)) return true;
else false;
}
return false;
}
int ANYLCLTRE::GenerateClassNamesFile(std::string labelMapFile, std::string objectDataFile) {
// Open the labelMapFile for reading
std::ifstream inputFile(labelMapFile, std::ios::in);
if (!inputFile.is_open()) {
std::cerr << "Error: Unable to open labelMapFile: " << labelMapFile << std::endl;
return -1; // Return an error code
}
// Open the objectDataFile for writing
std::ofstream outputFile(objectDataFile, std::ios::out);
if (!outputFile.is_open()) {
std::cerr << "Error: Unable to open objectDataFile: " << objectDataFile << std::endl;
inputFile.close();
return -2; // Return an error code
}
// Copy content from labelMapFile to objectDataFile
std::string line;
while (std::getline(inputFile, line)) {
outputFile << line << std::endl;
}
// Close the files
inputFile.close();
outputFile.close();
std::cout << "Content successfully copied from " << labelMapFile << " to " << objectDataFile << std::endl;
return 0; // Return success code
}
// We need to create a data folder in following structure
// data
// -cat1
// -cat2
// -cat3
// -CatergoryNames.txt
// data
// -train
// - cat1
// - cat2
// - cat3
// -test
// - cat1
// - cat2
// - cat3
// -val
// - cat1
// - cat2
// - cat3
bool ANYLCLTRE::UploadTrainingData(std::string projectName) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
//1. Check if the project folder exist, then create a data folder
std::string dataPath = projectPath + "\\data";
if (FolderExist(dataPath))DeleteFolder(dataPath);
fs::create_directory(dataPath);
std::string sourceDataPath = _workingDirectory + "\\data";
std::string sourceLabelPath = sourceDataPath + "\\CatergoryNames.txt";
if (!FileExist(sourceLabelPath)) return false;
//2. Generate the training dataset
GenerateTrainDataSet(sourceDataPath, dataPath, 0.7);
//3. Generate the classes.names file
std::string classesNamesFile = dataPath + "\\classes.names";
GenerateClassNamesFile(sourceLabelPath, classesNamesFile);
return FileExist(classesNamesFile);
}
bool ANYLCLTRE::CreateTrainingEngine(std::string projectName,
int experimentNumber,
int extractorType,
long numberStep,
int batchSize,
double learningRate)// extractorType =0: FAST, 1: ACCURATE; 2: VERY ACCURATE
{
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string experimentParentFolder = GetParentFolder(experimentPath);
std::string evaluationModelFile = experimentParentFolder + "\\yolo11n.pt";
if (!FolderExist(experimentParentFolder))fs::create_directory(experimentParentFolder);
if (!FileExist(evaluationModelFile)) CopyFile(yolo11nModelFile, evaluationModelFile);
//0. Check if the experiement folder exist
std::string zipExperimentPath = experimentPath + ".zip";
if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it
bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath);
if (!unzipResult) {
unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath);
}
if (unzipResult) {
DeleteFile(zipExperimentPath);
}
DeleteWeightsFiles(experimentPath);
}
//1. Create a brand new experiment folder
if (!FolderExist(experimentPath))fs::create_directory(experimentPath);
//2. Delete the old configuration files
std::string trainCfgFile = experimentPath + "\\train.cfg";
if (FileExist(trainCfgFile))DeleteFile(trainCfgFile);
//3. Copy classes.names file to the experiment folder
std::string classesNamesFile = projectPath + "\\data\\classes.names";
std::string experimentClassesNamesFile = experimentPath + "\\classes.names";
if (FileExist(classesNamesFile))CopyFile(classesNamesFile, experimentClassesNamesFile);
//4. Generate the configuration file
std::ofstream outFile(trainCfgFile);
outFile << "project_name=" << projectName << "\n";
outFile << "experiment_number=" << experimentNumber << "\n";
outFile << "extractor_type=" << extractorType << "\n";
outFile << "batch=" << batchSize << "\n";
outFile << "epochs=" << numberStep << "\n";
outFile << "lr0=" << learningRate << "\n";
outFile.close();
return true;
}
std::map<std::string, std::string> ANYLCLTRE::ExtractDataFromCfgFile(const std::string& filePath) {
std::map<std::string, std::string> data;
data.clear();
if (FileExist(filePath)) {
std::ifstream file(filePath);
if (!file.is_open()) {
return data;
}
std::string line;
while (std::getline(file, line)) {
std::istringstream iss(line);
std::string key, value;
if (std::getline(iss, key, '=') && std::getline(iss, value)) {
data[key] = value;
}
}
}
return data;
}
std::string ANYLCLTRE::GenerateTrainingCommand(std::string projectName, int experimentNumber, int extractorType) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string trainScript = _engineDirectory + "\\yolo.exe";
std::string trainCfgFile = experimentPath + "\\train.cfg";
std::string projectDataPath = projectPath + "\\data";
std::string preTrainWeightFile = experimentPath + "\\train_last.pt";
// Override the preTrainWeightFile if the experiment folder is already zipped
std::string zipExperimentPath = experimentPath + ".zip";
std::string experimentParentFolder = GetParentFolder(experimentPath);
if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it
bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath);
if (!unzipResult) {
unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath);
}
if (unzipResult) {
DeleteFile(zipExperimentPath);
}
}
//Copy classes.names file to the experiment folder
std::string classesNamesFile = projectPath + "\\data\\classes.names";
std::string classNamesFile = experimentPath + "\\classes.names";
if (FileExist(classNamesFile))DeleteFile(classNamesFile);
if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile);
// Get the folder that contain best.pt file
std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath);
// Check if the last train weight file is exist
std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName;
std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt";
//// Check if the last train weight file is exist
//std::string experimentResultPath = experimentPath + "\\" + _projectName;
//std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt";
if (FileExist(lastTrainWeightFile)) {
CopyFile(lastTrainWeightFile, preTrainWeightFile);
DeleteFolder(experimentResultPath);
}
if (!FileExist(preTrainWeightFile)) {
switch (extractorType) {// then use templae
case 0:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8n.pt";
break;
case 1:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8m.pt";
break;
case 2:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8x.pt";
break;
default:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8n.pt";
break;
}
}
if (!FileExist(trainScript)) this->_logger.LogError("ANYLCLTRE::GenerateTrainingCommand", "ANSCLTRE Engine is not installed", __FILE__, __LINE__);
if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLCLTRE::GenerateTrainingCommand", "Missing train.cfg file", __FILE__, __LINE__);
if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLCLTRE::GenerateTrainingCommand", "Missing pre-trained weight file", __FILE__, __LINE__);
if (!FolderExist(projectDataPath)) this->_logger.LogError("ANYLCLTRE::GenerateTrainingCommand", "Missing data yalm file", __FILE__, __LINE__);
if ((FileExist(trainScript)) &&
(FileExist(trainCfgFile)) &&
(FileExist(preTrainWeightFile)) &&
(FolderExist(projectDataPath))) {
// Extract information from train.cfg file
std::string epochs = "2000";
std::string learningRate = "0.01";
std::string batch = "-1"; // auto batch size
std::map<std::string, std::string> cfgMapData = ExtractDataFromCfgFile(trainCfgFile);
if (cfgMapData.size() > 0) {
epochs = cfgMapData.at("epochs");
learningRate = cfgMapData.at("lr0");
batch = cfgMapData.at("batch");
}
std::string trainCommand = trainScript +
" classify train model=" + "\"" + preTrainWeightFile + "\"" +
" data=" + "\"" + projectDataPath + "\"" +
" workers=8" +
" batch=" + batch +
" device=0" +
" epochs=" + epochs +
" plots=False" +
" patience=0" +
" name=" + _projectName +
" project=" + "\"" + experimentPath + "\"";
return trainCommand;
}
this->_logger.LogError("ANYLCLTRE::GenerateTrainingCommand", "Missing configuration files", __FILE__, __LINE__);
return "";
}
else {
this->_logger.LogError("ANYLCLTRE::GenerateTrainingCommand", "project path is not exist", __FILE__, __LINE__);
return "";
}
}
std::string ANYLCLTRE::GenerateCustomTrainingCommand(std::string projectName, int experimentNumber, int extractorType, std::string pretrainedModel) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string trainScript = _engineDirectory + "\\yolo.exe";
std::string trainCfgFile = experimentPath + "\\train.cfg";
std::string projectDataPath = projectPath + "\\data";
std::string preTrainWeightFile = pretrainedModel;
if (!FileExist(preTrainWeightFile)) preTrainWeightFile = experimentPath + "\\train_last.pt";
// Override the preTrainWeightFile if the experiment folder is already zipped
std::string zipExperimentPath = experimentPath + ".zip";
std::string experimentParentFolder = GetParentFolder(experimentPath);
if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it
bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath);
if (!unzipResult) {
unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath);
}
if (unzipResult) {
DeleteFile(zipExperimentPath);
}
}
//Copy classes.names file to the experiment folder
std::string classesNamesFile = projectPath + "\\data\\classes.names";
std::string classNamesFile = experimentPath + "\\classes.names";
if (FileExist(classNamesFile))DeleteFile(classNamesFile);
if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile);
std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath);
// Check if the last train weight file is exist
std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName;
std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt";
if (FileExist(lastTrainWeightFile)) {
CopyFile(lastTrainWeightFile, preTrainWeightFile);
DeleteFolder(experimentResultPath);
}
if (!FileExist(preTrainWeightFile)) {
switch (extractorType) {// then use templae
case 0:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8n.pt";
break;
case 1:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8m.pt";
break;
case 2:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8x.pt";
break;
default:
preTrainWeightFile = _modelTemplateDirectory + "\\cl\\cl8n.pt";
break;
}
}
if (!FileExist(trainScript)) this->_logger.LogError("ANYLCLTRE::GenerateCustomTrainingCommand", "ANSCLTRE Engine is not installed", __FILE__, __LINE__);
if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLCLTRE::GenerateCustomTrainingCommand", "Missing train.cfg file", __FILE__, __LINE__);
if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLCLTRE::GenerateCustomTrainingCommand", "Missing pre-trained weight file", __FILE__, __LINE__);
if ((FileExist(trainScript)) &&
(FileExist(trainCfgFile)) &&
(FileExist(preTrainWeightFile)) &&
(FolderExist(projectDataPath))) {
// Extract information from train.cfg file
std::string epochs = "2000";
std::string learningRate = "0.01";
std::string batch = "-1"; // auto batch size
std::map<std::string, std::string> cfgMapData = ExtractDataFromCfgFile(trainCfgFile);
if (cfgMapData.size() > 0) {
epochs = cfgMapData.at("epochs");
learningRate = cfgMapData.at("lr0");
batch = cfgMapData.at("batch");
}
std::string trainCommand = trainScript +
" classify train model=" + "\"" + preTrainWeightFile + "\"" +
" data=" + "\"" + projectDataPath + "\"" +
" workers=8" +
" batch=" + batch +
" device=0" +
" epochs=" + epochs +
" plots=False"
" patience=0" +
" name=" + _projectName +
" project=" + "\"" + experimentPath + "\"";
return trainCommand;
}
this->_logger.LogError("ANYLCLTRE::GenerateCustomTrainingCommand", "Missing configuration files", __FILE__, __LINE__);
return "";
}
else {
this->_logger.LogError("ANYLCLTRE::GenerateCustomTrainingCommand", "project path is not exist", __FILE__, __LINE__);
return "";
}
}
std::string ANYLCLTRE::GenerateEvaluateModelCommand(std::string projectName, int experimentNumber, bool& zipModelExist) {
_projectName = Space2Underscore(projectName);
zipModelExist = false;
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string trainScript = _engineDirectory + "\\yolo.exe";
std::string trainCfgFile = experimentPath + "\\train.cfg";
std::string preTrainWeightFile = experimentPath + "\\train_last.pt";
std::string projectDataPath = projectPath + "\\data";
std::string runFolder = experimentPath + "\\run";
std::string comelRunFolder = experimentPath + "\\.cometml-runs";
// Override the preTrainWeightFile if the experiment folder is already zipped
std::string zipExperimentPath = experimentPath + ".zip";
std::string experimentParentFolder = GetParentFolder(experimentPath);
if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it
bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath);
if (!unzipResult) {
unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath);
}
if (unzipResult) {
zipModelExist = true;
}
}
//Copy classes.names file to the experiment folder
std::string classesNamesFile = projectPath + "\\data\\classes.names";
std::string classNamesFile = experimentPath + "\\classes.names";
if (FileExist(classNamesFile))DeleteFile(classNamesFile);
if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile);
if (FolderExist(runFolder))DeleteFolder(runFolder);
if (FolderExist(comelRunFolder))DeleteFolder(comelRunFolder);
std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath);
// Check if the last train weight file is exist
std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName;
std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt";
if (FileExist(lastTrainWeightFile)) {
CopyFile(lastTrainWeightFile, preTrainWeightFile);
DeleteFolder(experimentResultPath);
}
if (!FileExist(trainScript)) this->_logger.LogError("ANYLCLTRE::GenerateEvaluateModelCommand", "ANYLCLTRE Engine is not installed", __FILE__, __LINE__);
if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLCLTRE::GenerateEvaluateModelCommand", "Missing train.cfg file", __FILE__, __LINE__);
if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLCLTRE::GenerateEvaluateModelCommand", "Missing pre-trained weight file", __FILE__, __LINE__);
if (!FileExist(classNamesFile)) this->_logger.LogError("ANYLCLTRE::GenerateEvaluateModelCommand", "Missing classes.names file", __FILE__, __LINE__);
if ((FileExist(trainScript)) &&
(FileExist(trainCfgFile)) &&
(FileExist(preTrainWeightFile)) &&
(FileExist(classNamesFile))) {
std::string evaluateCommand = trainScript +
" classify val model=" + "\"" + preTrainWeightFile + "\"" +
" data=" + "\"" + projectDataPath + "\"" +
" split=val" +
" workers=8" +
" batch=8" +
" device=0" +
" conf=0.25" +
" plots=False" +
" mlflow=False" +
" save_json=False";
return evaluateCommand;
}
this->_logger.LogError("ANYLCLTRE::GenerateEvaluateModelCommand", "Missing configuration files", __FILE__, __LINE__);
return "";
}
else {
this->_logger.LogError("ANYLCLTRE::GenerateEvaluateModelCommand", "project path is not exist", __FILE__, __LINE__);
return "";
}
}
bool ANYLCLTRE::DownloadModel(std::string projectName, int experimentNumber, std::string destZipDirectory, int modelMode) {
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
//0. Get all files in the experiment folder
std::string trainScript = _engineDirectory + "\\yolo.exe";
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string zipExperimentPath = experimentPath + ".zip";
std::string trainCfgFile = experimentPath + "\\train.cfg";
std::string runFolder = experimentPath + "\\run";
int extractorType = 0;
if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it
if (ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath))
{
DeleteFile(zipExperimentPath);
}
else {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "Can not unzip model zip file to extract information. Invalid password.", __FILE__, __LINE__);
return false;
}
}
//Copy classes.names file to the experiment folder
std::string classesNamesFile = projectPath + "\\data\\classes.names";
std::string classNamesFile = experimentPath + "\\classes.names";
if (FileExist(classNamesFile))DeleteFile(classNamesFile);
if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile);
// Extract information from train.cfg file
std::map<std::string, std::string> cfgMapData = ExtractDataFromCfgFile(trainCfgFile);
if (cfgMapData.size() > 0) {
extractorType = std::stoi(cfgMapData.at("extractor_type"));
}
std::string exportDirectory = projectPath + "\\exports";
if (!FolderExist(exportDirectory))fs::create_directory(exportDirectory);
std::string exportExperimentFolder = exportDirectory + "\\" + projectName + "_" + std::to_string(experimentNumber);
std::string exportExperimentZip = exportExperimentFolder + ".zip";
if (FileExist(exportExperimentZip))DeleteFile(exportExperimentZip);
if (!FolderExist(exportExperimentFolder))fs::create_directory(exportExperimentFolder);
// Get the folder that contain best.pt file
std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath);
// Check if the last train weight file is exist
std::string preTrainWeightFile = experimentPath + "\\train_last.pt";
std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName;
std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt";
if (FileExist(lastTrainWeightFile)) {
if (FileExist(preTrainWeightFile))DeleteFile(preTrainWeightFile);
CopyFile(lastTrainWeightFile, preTrainWeightFile);
DeleteFolder(experimentResultPath);
}
if (!FileExist(preTrainWeightFile)) {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "Missing pre-trained weight file", __FILE__, __LINE__);
return false;
}
// 1. Get the names of classes.names
std::ifstream inFile(classNamesFile);
if (!inFile.is_open()) {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "cannot open file to save model information.", __FILE__, __LINE__);
return false;
}
std::vector<std::string> classesNames;
std::string line;
while (std::getline(inFile, line)) {
classesNames.push_back(line);
}
inFile.close();
// 2. Copy the classes.names file to the export folder
std::string exportClassesNameFile = exportExperimentFolder + "\\classes.names";
CopyFile(classNamesFile, exportClassesNameFile);
// 3. Create Categories folder
std::string categoriesFolder = exportExperimentFolder + "\\Categories";
if (FolderExist(categoriesFolder))DeleteFolder(categoriesFolder);
fs::create_directory(categoriesFolder);
for (size_t i = 0; i < classesNames.size(); ++i) {
std::ofstream outFile(categoriesFolder + "/" + classesNames[i] + "." + std::to_string(i + 1));
outFile.close();
}
//4. Create a text file name yolo.txt
if (modelMode == 0) { // This is the tensorRT onnx model
std::string yoloTxtFile = exportExperimentFolder + "\\tensorrtcl.txt";
std::ofstream outFile(yoloTxtFile);
outFile.close();
std::string exportCommand = trainScript +
" export model=" + preTrainWeightFile +
" format=onnx" +
" dynamic=True " +
" opset=12" +
" simplify=False";
if (!FileExist(preTrainWeightFile)) {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "Pretrain file does not exist.", __FILE__, __LINE__);
return false;
}
std::string onnxFile = experimentPath + "\\train_last.onnx";
if (FileExist(onnxFile))DeleteFile(onnxFile);
ExecuteCommand(exportCommand.c_str());
if (!FileExist(onnxFile)) {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "cannot generate onnx file.", __FILE__, __LINE__);
return false;
}
// Copy the onnx file to the export folder
std::string exportOnnxFile = exportExperimentFolder + "\\train_last.onnx";
CopyFile(onnxFile, exportOnnxFile);
DeleteFile(onnxFile);
}
else {
std::string yoloTxtFile = exportExperimentFolder + "\\openvinocl.txt";
std::ofstream outFile(yoloTxtFile);
outFile.close();
// Need to run export command to get the openvino files
// then access the openvino folder to get the xml and bin files
// renname bin and xml file to train_last.bin and train_last.xml
// then delete the pt file
// then delete the openvino folder
std::string pythonDir = GetParentFolder(_engineDirectory);
std::string pythonScript = pythonDir + "\\python.exe";
std::string exportCommand =trainScript +
" export model=" + preTrainWeightFile +
" half = False" +
" format=openvino" +
" simplify=False";
if (!FileExist(preTrainWeightFile)) {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "Pretrain file does not exist.", __FILE__, __LINE__);
return false;
}
ExecuteCommand(exportCommand.c_str());
std::string xmlTrainedModelFile = exportExperimentFolder + "\\train_last.xml";
std::string binTrainedModelFile = exportExperimentFolder + "\\train_last.bin";
if (FileExist(xmlTrainedModelFile)) DeleteFile(xmlTrainedModelFile);
if (FileExist(binTrainedModelFile)) DeleteFile(binTrainedModelFile);
std::string outputExportedFolder = experimentPath + "\\train_last_openvino_model";
std::string xmlFile = outputExportedFolder + "\\train_last.xml";
std::string binFile = outputExportedFolder + "\\train_last.bin";
if (!FileExist(xmlFile)) {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "cannot generate openvino file.", __FILE__, __LINE__);
return false;
}
CopyFile(xmlFile, xmlTrainedModelFile);
CopyFile(binFile, binTrainedModelFile);
DeleteFolder(outputExportedFolder);
}
// Delete the run folder
if (FolderExist(runFolder))DeleteFolder(runFolder);
//6. Zip the experiment folder
if (ZipFolderWithPassword(experimentPath.c_str(), zipExperimentPath.c_str(), _zipPassword.c_str())) {
DeleteFolder(experimentPath);
}
// 7. Zip the Experiment folder
if (ZipFolderWithPassword(exportExperimentFolder.c_str(), exportExperimentZip.c_str(), _zipPassword.c_str())) {
DeleteFolder(exportExperimentFolder);
std::string destZipFilePath = destZipDirectory + "\\ANS_" + projectName + "(GPU)_" + std::to_string(experimentNumber) + ".zip";
if (modelMode > 0)destZipFilePath = destZipDirectory + "\\ANS_" + projectName + "(CPU)_" + std::to_string(experimentNumber) + ".zip";
//std::string destZipFilePath = destZipDirectory + "\\" + projectName + "_" + std::to_string(experimentNumber) + ".zip";
CopyFile(exportExperimentZip, destZipFilePath);
if (FileExist(destZipFilePath)) {
//DeleteFile(exportExperimentZip);
return true;
}
return false;
}
this->_logger.LogError("ANYLCLTRE::DownloadModel", "Can not unzip model zip file to extract information. Invalid password.", __FILE__, __LINE__);
return false;
}
else {
this->_logger.LogError("ANYLCLTRE::DownloadModel", "Invalid project path.", __FILE__, __LINE__);
return false;
}
}
std::string ANYLCLTRE::EvaluateModel(std::string projectName, int experimentNumber) {
bool zipModelExist = false;
_projectName = Space2Underscore(projectName);
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string zipExperimentPath = experimentPath + ".zip";
std::string preTrainWeightFile = experimentPath + "\\train_last.pt";
std::string evaluateCommand = GenerateEvaluateModelCommand(_projectName, experimentNumber, zipModelExist);
if (evaluateCommand.empty()) {
this->_logger.LogError("ANSODTRE::EvaluateModel", "evaluateCommand is empty", __FILE__, __LINE__);
return "";
}
if (!FileExist(preTrainWeightFile)) {
this->_logger.LogError("ANSODTRE::EvaluateModel", "pre-trained model is not exist", __FILE__, __LINE__);
return "";
}
return evaluateCommand;
}
else {
this->_logger.LogError("ANYLCLTRE::EvaluateModel", "Invalid project path.", __FILE__, __LINE__);
return "";
}
}
std::string ANYLCLTRE::ParseTrainingResults(std::string trainingResults) {
if (trainingResults.empty()) return "step: 0; loss=0.00; map=0.00";
try {
// Regex pattern to extract the first integer from "1/2000"
std::regex epoch_regex(R"((\d+)/\d+)");
std::smatch epoch_match;
std::string epoch = "0";
std::string loss = "0.00";
std::string map = "0.00";
if (std::regex_search(trainingResults, epoch_match, epoch_regex)) {
epoch = epoch_match[1].str();
}
// Regex to find the line containing "all" and then extract the last two items (0.994 and 0.81) from that line
std::regex line_regex(R"(all.*?(\d+\.\d+)\s+(\d+\.\d+)$)");
std::smatch line_match;
if (std::regex_search(trainingResults, line_match, line_regex)) {
loss = line_match[1].str();
map = line_match[2].str();
}
std::ostringstream status;
status << "step: " << epoch << "; loss=" << loss << "; map=" << map;
std::string result = status.str();
std::cout << "result:" << result << std::endl;
return result;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANYLCLTRE::ParseTrainingResults", e.what(), __FILE__, __LINE__);
return "step: 0; loss=0.00; map=0.00";
}
}
bool ANYLCLTRE::CheckEngineStatus() {
const char* targetProcess = "yolo.exe"; // Replace with the target process name
if (IsProcessRunning(targetProcess)) {
return true;
}
else {
return false;
}
}
bool ANYLCLTRE::CheckEngine() {
std::string inferenceCommand = "C:\\ProgramData\\ANSCENTER\\Python311\\Scripts\\yolo.exe detect predict model = \"C:\\ProgramData\\ANSCENTER\\Python311\\Lib\\site-packages\\data\\od\\od8m.pt\" source = \"C:\\ProgramData\\ANSCENTER\\Shared\\test.jpg\" save = False";
std::string result = ExecuteCommand(inferenceCommand.c_str());
if (result.empty()) {
return false;
}
else {
std::string target = "640x480 4 persons, 1 bus";
// Check if the target text is found in the input text
if (result.find(target) != std::string::npos) {
std::cout << "Engine is installed correctly." << std::endl;
return true; // Text is found
}
else {
std::cout << "Engine is not installed correctly." << std::endl;
return false; // Text is not found
}
}
}
//Private functions
std::string ANYLCLTRE::ParseMAP(const std::string& text) {
// Updated regex pattern for "all <float> <integer>"
std::regex valuePattern(R"(all\s+([\d\.]+)\s+\d+)");
std::smatch matches;
// Search for the pattern
if (std::regex_search(text, matches, valuePattern) && matches.size() > 1) {
return matches[1].str(); // Return the captured floating-point number
}
else {
return "0.00"; // Default return value if no match
}
}
std::string ANYLCLTRE::AdjustRootDirectory(const std::string& root) {
std::filesystem::path rootPath(root);
std::vector<std::filesystem::path> subdirectories;
for (const auto& entry : std::filesystem::directory_iterator(rootPath)) {
if (entry.is_directory()) {
subdirectories.push_back(entry.path());
}
}
if (subdirectories.size() == 1) {
return subdirectories[0].string();
}
else {
return root;
}
}
std::string ANYLCLTRE::GenerateClassNames(std::string labelMapFile) {
std::ifstream inFile(labelMapFile);
if (!inFile.is_open()) {
return "[]";
}
std::string classesNames = "[";
std::string line;
std::regex namePattern("name:'(.*)'");
int fileLine = 0;
while (std::getline(inFile, line)) {
std::smatch matches;
if (std::regex_search(line, matches, namePattern) && matches.size() > 1) {
classesNames += matches[1].str() + ",";
fileLine++;
}
}
inFile.close();
// Remove the last comma if there is one
if (classesNames.back() == ',') {
classesNames.pop_back();
}
classesNames += "]";
return classesNames;
}
void ANYLCLTRE::GenerateTrainDataSet(const std::string& sourceFolder, std::string projectDir, double trainRatio) {
// 1. Create train, test, and val folders
std::string trainFolder = projectDir + "\\train";
std::string testFolder = projectDir + "\\test";
std::string valFolder = projectDir + "\\val";
if (FolderExist(trainFolder)) DeleteFolder(trainFolder);
if (FolderExist(testFolder)) DeleteFolder(testFolder);
if (FolderExist(valFolder)) DeleteFolder(valFolder);
fs::create_directory(trainFolder);
fs::create_directory(testFolder);
fs::create_directory(valFolder);
// 2. Iterate through sub-folders in the source folder
for (const auto& entry : fs::directory_iterator(sourceFolder)) {
if (!entry.is_directory()) continue;
std::string subFolderName = entry.path().filename().string();
std::string sourceSubFolder = entry.path().string();
std::string trainSubFolder = trainFolder + "\\" + subFolderName;
std::string testSubFolder = testFolder + "\\" + subFolderName;
std::string valSubFolder = valFolder + "\\" + subFolderName;
fs::create_directory(trainSubFolder);
fs::create_directory(testSubFolder);
fs::create_directory(valSubFolder);
// 3. Collect all image files from the current sub-folder
std::vector<fs::path> imageFiles;
for (const auto& file : fs::directory_iterator(sourceSubFolder)) {
if (file.is_regular_file()) {
// Match image files by common extensions
std::string extension = file.path().extension().string();
std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower);
if (extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".bmp" || extension == ".tiff") {
imageFiles.push_back(file.path());
}
}
}
// Shuffle the files
std::random_device rd;
std::mt19937 g(rd());
std::shuffle(imageFiles.begin(), imageFiles.end(), g);
// Split the files into train, test, and val sets
size_t totalFiles = imageFiles.size();
size_t trainCount = static_cast<size_t>(totalFiles * trainRatio);
size_t valCount = static_cast<size_t>((totalFiles - trainCount) / 2);
size_t testCount = totalFiles - trainCount - valCount;
auto copyFiles = [](const std::vector<fs::path>& files, size_t start, size_t count, const std::string& destinationFolder) {
for (size_t i = start; i < start + count; ++i) {
if (i >= files.size()) break;
const fs::path& src = files[i];
fs::copy(src, destinationFolder + "\\" + src.filename().string(), fs::copy_options::overwrite_existing);
}
};
copyFiles(imageFiles, 0, trainCount, trainSubFolder);
copyFiles(imageFiles, trainCount, testCount, testSubFolder);
copyFiles(imageFiles, trainCount + testCount, valCount, valSubFolder);
}
}
bool ANYLCLTRE::GenerateConfigFile(const std::string& templateFile, const std::string& outputFile, const std::map<std::string, std::string>& replacements) {
std::ifstream inFile(templateFile);
std::ofstream outFile(outputFile);
if (!inFile.is_open() || !outFile.is_open()) {
return false;
}
std::string line;
while (std::getline(inFile, line)) {
for (const auto& replacement : replacements) {
size_t pos = line.find("$" + replacement.first);
while (pos != std::string::npos) {
line.replace(pos, replacement.first.size() + 1, replacement.second);
pos = line.find("$" + replacement.first, pos + replacement.second.size());
}
}
outFile << line << "\n";
}
inFile.close();
outFile.close();
return FileExist(outputFile);
}
int ANYLCLTRE::ReadClassesValue(const std::string& filePath) {
std::ifstream inFile(filePath);
if (!inFile.is_open()) {
return 0;
}
std::ofstream outFile(filePath, std::ios_base::app);
if (!outFile.is_open()) {
return 0;
}
std::string currentFolderName = GetParentFolder(filePath);
outFile << "backup = " << currentFolderName << "\n";
outFile.close();
std::string line;
while (std::getline(inFile, line)) {
if (line.find("classes") == 0) {
size_t pos = line.find("=");
if (pos != std::string::npos) {
std::string classesValue = line.substr(pos + 1);
inFile.close();
return std::stoi(classesValue);
}
}
}
inFile.close();
return 0;
}
void ANYLCLTRE::DeleteWeightsFiles(const std::string& folderPath) {
for (const auto& entry : std::filesystem::directory_iterator(folderPath)) {
if (entry.is_regular_file() && entry.path().extension() == ".weights" && entry.path().filename() != "train_last.weights") {
std::filesystem::remove(entry.path());
}
}
}
std::string ANYLCLTRE::GenerateTestCommand(std::string projectName, int experimentNumber, bool& zipModelExist, const std::string& testDataFolder) {
_projectName = Space2Underscore(projectName);
zipModelExist = false;
std::string projectPath = FindFilePathInFileList(_projects, _projectName);
if (FolderExist(projectPath)) {
std::string experimentPath = projectPath + "\\experiments\\" + std::to_string(experimentNumber);
std::string trainScript = _engineDirectory + "\\yolo.exe";
std::string trainCfgFile = experimentPath + "\\train.cfg";
std::string preTrainWeightFile = experimentPath + "\\train_last.pt";
std::string projectDataPath = projectPath + "\\data";
std::string runFolder = experimentPath + "\\run";
std::string comelRunFolder = experimentPath + "\\.cometml-runs";
// Override the preTrainWeightFile if the experiment folder is already zipped
std::string zipExperimentPath = experimentPath + ".zip";
std::string experimentParentFolder = GetParentFolder(experimentPath);
if (FileExist(zipExperimentPath)) { // If the experiment folder is already zipped, we need to unzip it
bool unzipResult = ExtractProtectedZipFile(zipExperimentPath, _zipPassword, std::to_string(experimentNumber), experimentPath);
if (!unzipResult) {
unzipResult = ExtractProtectedZipFile(zipExperimentPath, _modelZipPassword, std::to_string(experimentNumber), experimentPath);
}
if (unzipResult) {
zipModelExist = true;
}
}
//Copy classes.names file to the experiment folder
std::string classesNamesFile = projectPath + "\\data\\classes.names";
std::string classNamesFile = experimentPath + "\\classes.names";
if (FileExist(classNamesFile))DeleteFile(classNamesFile);
if (FileExist(classesNamesFile))CopyFile(classesNamesFile, classNamesFile);
if (FolderExist(runFolder))DeleteFolder(runFolder);
if (FolderExist(comelRunFolder))DeleteFolder(comelRunFolder);
std::string weightFolder = GetLatestValidFolderAndDeleteOthers(projectName, experimentPath);
// Check if the last train weight file is exist
std::string experimentResultPath = weightFolder;// experimentPath + "\\" + _projectName;
std::string lastTrainWeightFile = experimentResultPath + "\\weights\\best.pt";
if (FileExist(lastTrainWeightFile)) {
CopyFile(lastTrainWeightFile, preTrainWeightFile);
DeleteFolder(experimentResultPath);
}
if (!FileExist(trainScript)) this->_logger.LogError("ANYLCLTRE::GenerateTestCommand", "ANYLCLTRE Engine is not installed", __FILE__, __LINE__);
if (!FileExist(trainCfgFile)) this->_logger.LogError("ANYLCLTRE::GenerateTestCommand", "Missing train.cfg file", __FILE__, __LINE__);
if (!FileExist(preTrainWeightFile)) this->_logger.LogError("ANYLCLTRE::GenerateTestCommand", "Missing pre-trained weight file", __FILE__, __LINE__);
if (!FileExist(classNamesFile)) this->_logger.LogError("ANYLCLTRE::GenerateTestCommand", "Missing classes.names file", __FILE__, __LINE__);
std::string testDataPath = testDataFolder + "\\data";
if ((FileExist(trainScript)) &&
(FileExist(trainCfgFile)) &&
(FileExist(preTrainWeightFile)) &&
(FileExist(classNamesFile))) {
std::string evaluateCommand = trainScript +
" classify val model=" + "\"" + preTrainWeightFile + "\"" +
" data=" + "\"" + testDataPath + "\"" +
" split=val" +
" workers=8" +
" batch=8" +
" device=0" +
" conf=0.25" +
" plots=False" +
" mlflow=False" +
" save_json=False";
return evaluateCommand;
}
this->_logger.LogError("ANYLCLTRE::GenerateTestCommand", "Missing configuration files", __FILE__, __LINE__);
return "";
}
else {
this->_logger.LogError("ANYLCLTRE::GenerateTestCommand", "project path is not exist", __FILE__, __LINE__);
return "";
}
}
}
Reference in New Issue Copy Permalink