1953 lines
73 KiB
C++
1953 lines
73 KiB
C++
|
|
#pragma once
|
|||
|
|
#include "ANSODEngine.h"
|
|||
|
|
#include "ANSYOLOOD.h"
|
|||
|
|
#include "ANSTENSORRTOD.h"
|
|||
|
|
#include "ANSTENSORRTCL.h"
|
|||
|
|
#include "ANSOPENVINOCL.h"
|
|||
|
|
#include "ANSOPENVINOOD.h"
|
|||
|
|
#include "ANSYOLOV10RTOD.h"
|
|||
|
|
#include "ANSYOLOV10OVOD.h"
|
|||
|
|
#include "ANSCUSTOMDetector.h"
|
|||
|
|
#include "ANSFD.h"
|
|||
|
|
#include "ANSANOMALIB.h"
|
|||
|
|
#include "ANSPOSE.h"
|
|||
|
|
#include "ANSSAM.h"
|
|||
|
|
#include <pipelines/metadata.h>
|
|||
|
|
#include <models/input_data.h>
|
|||
|
|
#include "utils/visualizer.hpp"
|
|||
|
|
#include <turbojpeg.h>
|
|||
|
|
#include <vector>
|
|||
|
|
#include <map>
|
|||
|
|
#include <string>
|
|||
|
|
#define USEONNXOV
|
|||
|
|
//#define ISVALIDFACE_DEBUG
|
|||
|
|
namespace ANSCENTER
|
|||
|
|
{
|
|||
|
|
void ANSFDBase::CheckLicense() {
|
|||
|
|
try {
|
|||
|
|
_licenseValid = ANSCENTER::ANSLicenseHelper::LicenseVerification(_licenseKey, 1004, "ANSFR");//Default productId=1004
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::CheckLicense()", e.what(), __FILE__, __LINE__);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
ModelConfig ANSFDBase::GetModelConfig() {
|
|||
|
|
return _modelConfig;
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)
|
|||
|
|
{
|
|||
|
|
Cleanup();
|
|||
|
|
try {
|
|||
|
|
ANSCENTER::ANSLibsLoader::Initialize();
|
|||
|
|
_modelFolder = "";
|
|||
|
|
_modelConfigFile = "";
|
|||
|
|
_modelFolder.clear();
|
|||
|
|
_modelConfigFile.clear();
|
|||
|
|
// 0. Check if the modelZipFilePath exist?
|
|||
|
|
if (!FileExist(modelZipFilePath)) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::LoadModel", "Model zip file is not exist", __FILE__, __LINE__);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 1. Unzip model zip file to a special location with folder name as model file (and version)
|
|||
|
|
std::string outputFolder;
|
|||
|
|
std::vector<std::string> passwordArray;
|
|||
|
|
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
|
|||
|
|
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
|
|||
|
|
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
|
|||
|
|
passwordArray.push_back("AnsCustomModels20@$");
|
|||
|
|
passwordArray.push_back("AnsDemoModels20@!");
|
|||
|
|
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
|
|||
|
|
//this->_logger.LogDebug("ANSFDBase::LoadModel. Model name", modelName, __FILE__, __LINE__);
|
|||
|
|
size_t vectorSize = passwordArray.size();
|
|||
|
|
for (size_t i = 0; i < vectorSize; i++) {
|
|||
|
|
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
|
|||
|
|
break; // Break the loop when the condition is met.
|
|||
|
|
}
|
|||
|
|
// 2. Check if the outputFolder exist
|
|||
|
|
if (!std::filesystem::exists(_modelFolder)) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadModel. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
|
|||
|
|
return false; // That means the model file is not exist or the password is not correct
|
|||
|
|
}
|
|||
|
|
// 3. Check if the model has the configuration file
|
|||
|
|
std::string modelConfigName = "model_config.json";
|
|||
|
|
_modelConfigFile = CreateFilePath(_modelFolder, modelConfigName);
|
|||
|
|
_facelivenessEngineValid = false;
|
|||
|
|
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel.zip";
|
|||
|
|
if (FileExist(livenessModelFile)) {
|
|||
|
|
_facelivenessEngineValid = InitializeLivenessModel(_licenseKey, livenessModelFile, modelZipPassword);
|
|||
|
|
}
|
|||
|
|
if (_facelivenessEngineValid) {
|
|||
|
|
//Initialize face tracker
|
|||
|
|
CreateANSMOTHandle(&_faceTracker, 1);
|
|||
|
|
std::string params = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
|
|||
|
|
UpdateANSMOTParameters(&_faceTracker, params.c_str());
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
//Initialize tv tracker
|
|||
|
|
CreateANSMOTHandle(&_tvTracker, 1);
|
|||
|
|
std::string paramsTv = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
|
|||
|
|
UpdateANSMOTParameters(&_tvTracker, paramsTv.c_str());
|
|||
|
|
#endif
|
|||
|
|
}
|
|||
|
|
return true; // Initialization successful
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::LoadModel", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
|
|||
|
|
ANSCENTER::ANSLibsLoader::Initialize();
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
if (_useTvDetector) {
|
|||
|
|
if (!this->_tvDetector->OptimizeModel(fp16, optimizedModelFolder)) return false;
|
|||
|
|
// Zip the optimized model folder
|
|||
|
|
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel_Optimized.zip";
|
|||
|
|
if (FileExist(livenessModelFile)) return true;
|
|||
|
|
if (FolderExist(optimizedModelFolder)) {
|
|||
|
|
std::string modelZipPassword = "Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==";
|
|||
|
|
return ZipFolderWithPassword(optimizedModelFolder.c_str(), livenessModelFile.c_str(), modelZipPassword.c_str());
|
|||
|
|
}
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
else return true;
|
|||
|
|
#else
|
|||
|
|
return true;
|
|||
|
|
#endif
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
|
|||
|
|
// Call Cleanup to ensure any existing resources are released before initialization
|
|||
|
|
Cleanup();
|
|||
|
|
try {
|
|||
|
|
ANSCENTER::ANSLibsLoader::Initialize();
|
|||
|
|
_licenseKey = licenseKey;
|
|||
|
|
_licenseValid = false;
|
|||
|
|
_modelFolder.clear();
|
|||
|
|
_modelConfigFile.clear();
|
|||
|
|
_imageProcessingModelFile.clear();
|
|||
|
|
|
|||
|
|
//1. License check
|
|||
|
|
CheckLicense();
|
|||
|
|
if (!_licenseValid) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::Initialize", "Invalid License", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//2. Check if model zip file exists
|
|||
|
|
if (!FileExist(modelZipFilePath)) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::Initialize", "Model zip file does not exist", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//3. Unzip model file
|
|||
|
|
//_engineType = ANSLicenseHelper::CheckHardwareInformation();
|
|||
|
|
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
|
|||
|
|
std::vector<std::string> passwordArray = { modelZipPassword,
|
|||
|
|
"Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==",
|
|||
|
|
"3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==",
|
|||
|
|
"AnsCustomModels20@$",
|
|||
|
|
"AnsDemoModels20@!" };
|
|||
|
|
bool extractionSuccess = false;
|
|||
|
|
|
|||
|
|
for (const auto& password : passwordArray) {
|
|||
|
|
if (ExtractPasswordProtectedZip(modelZipFilePath, password, modelName, _modelFolder, false)) {
|
|||
|
|
extractionSuccess = true;
|
|||
|
|
break; // Break on successful extraction
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!extractionSuccess || !std::filesystem::exists(_modelFolder)) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::Initialize. Output model folder does not exist", _modelFolder, __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
_modelConfigFile = CreateFilePath(_modelFolder, "model_config.json");
|
|||
|
|
|
|||
|
|
// 4. Load the image processing model
|
|||
|
|
//std::string imageProcessingModel = "C:\\ProgramData\\ANSCENTER\\Shared\\ANS_ImageProcessing_v1.0.zip";
|
|||
|
|
//if (FileExist(imageProcessingModel)) {
|
|||
|
|
// std::string imagemodelName = GetFileNameWithoutExtension(imageProcessingModel);
|
|||
|
|
// std::string _imageModelFolder;
|
|||
|
|
// for (const auto& password : passwordArray) {
|
|||
|
|
// if (ExtractPasswordProtectedZip(imageProcessingModel, password, imagemodelName, _imageModelFolder, false)) {
|
|||
|
|
// extractionSuccess = true;
|
|||
|
|
// break; // Break on successful extraction
|
|||
|
|
// }
|
|||
|
|
// }
|
|||
|
|
// _imageProcessingModelFile = CreateFilePath(_imageModelFolder, "upscale.xml");
|
|||
|
|
// ov::Core core;
|
|||
|
|
// if (FileExist(_imageProcessingModelFile)) {
|
|||
|
|
// std::string deviceName = GetOpenVINODevice();
|
|||
|
|
// try {
|
|||
|
|
// if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
|
|||
|
|
// {
|
|||
|
|
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(28, 28), ""));
|
|||
|
|
// model->setInputsPreprocessing(false, "", "");
|
|||
|
|
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig(deviceName, 1, "", 4), core);
|
|||
|
|
|
|||
|
|
// }
|
|||
|
|
// else {
|
|||
|
|
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(40, 40), ""));
|
|||
|
|
// model->setInputsPreprocessing(false, "", "");
|
|||
|
|
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig(deviceName, 1, "", 4), core);
|
|||
|
|
// }
|
|||
|
|
// }
|
|||
|
|
// catch (const std::exception& e) {
|
|||
|
|
// // Fall back to CPU if GPU is not available
|
|||
|
|
// if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
|
|||
|
|
// {
|
|||
|
|
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(28, 28), ""));
|
|||
|
|
// model->setInputsPreprocessing(false, "", "");
|
|||
|
|
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig("CPU", 1, "", 4), core);
|
|||
|
|
// }
|
|||
|
|
// else {
|
|||
|
|
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(40, 40), ""));
|
|||
|
|
// model->setInputsPreprocessing(false, "", "");
|
|||
|
|
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig("CPU", 1, "", 4), core);
|
|||
|
|
// }
|
|||
|
|
// }
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
//}
|
|||
|
|
// Initalize faceliveness model
|
|||
|
|
_facelivenessEngineValid = false;
|
|||
|
|
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel_Optimized.zip";
|
|||
|
|
if (!FileExist(livenessModelFile)) livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel.zip";
|
|||
|
|
|
|||
|
|
if (FileExist(livenessModelFile)) {
|
|||
|
|
_facelivenessEngineValid = InitializeLivenessModel(_licenseKey, livenessModelFile, modelZipPassword);
|
|||
|
|
}
|
|||
|
|
if (_facelivenessEngineValid) {
|
|||
|
|
//Initialize face tracker
|
|||
|
|
CreateANSMOTHandle(&_faceTracker, 1);
|
|||
|
|
std::string params = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
|
|||
|
|
UpdateANSMOTParameters(&_faceTracker, params.c_str());
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
//Initialize tv tracker
|
|||
|
|
CreateANSMOTHandle(&_tvTracker, 1);
|
|||
|
|
std::string paramsTv = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
|
|||
|
|
UpdateANSMOTParameters(&_tvTracker, paramsTv.c_str());
|
|||
|
|
#endif
|
|||
|
|
}
|
|||
|
|
return true; // Initialization successful
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::Initialize", e.what(), __FILE__, __LINE__);
|
|||
|
|
Cleanup(); // Ensure cleanup on failure
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
void ANSFDBase::Cleanup() {
|
|||
|
|
// Clear model paths and reset related data members
|
|||
|
|
_modelFolder.clear();
|
|||
|
|
_modelConfigFile.clear();
|
|||
|
|
_imageProcessingModelFile.clear();
|
|||
|
|
if (this->_faceTracker)
|
|||
|
|
{
|
|||
|
|
ReleaseANSMOTHandle(&_faceTracker);
|
|||
|
|
}
|
|||
|
|
if (_livenessSession) {
|
|||
|
|
delete _livenessSession;
|
|||
|
|
}
|
|||
|
|
if (_ortLivenessSessionOptions) {
|
|||
|
|
delete _ortLivenessSessionOptions;
|
|||
|
|
}
|
|||
|
|
if (_ortLivenessEnv) {
|
|||
|
|
delete _ortLivenessEnv;
|
|||
|
|
}
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
if (this->_tvTracker)
|
|||
|
|
{
|
|||
|
|
ReleaseANSMOTHandle(&_tvTracker);
|
|||
|
|
}
|
|||
|
|
if (_useTvDetector && this->_tvDetector) {
|
|||
|
|
this->_tvDetector->Destroy();
|
|||
|
|
this->_tvDetector.reset();
|
|||
|
|
}
|
|||
|
|
#endif
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
|
|||
|
|
Cleanup();
|
|||
|
|
try {
|
|||
|
|
ANSCENTER::ANSLibsLoader::Initialize();
|
|||
|
|
_licenseKey = licenseKey;
|
|||
|
|
_licenseValid = false;
|
|||
|
|
_modelFolder = modelFolder;
|
|||
|
|
_modelConfigFile = "";
|
|||
|
|
_modelConfigFile.clear();
|
|||
|
|
// 0. Check license
|
|||
|
|
CheckLicense();
|
|||
|
|
if (!_licenseValid) {
|
|||
|
|
this->_logger.LogError("LoadModelFromFolder::Initialize", "Invalid License", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
// 1. Check if the modelFolder exist?
|
|||
|
|
if (!FolderExist(modelFolder)) {
|
|||
|
|
this->_logger.LogFatal("LoadModelFromFolder::Initialize", "Model zip file is not exist", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
// 3. Check if the model has the configuration file
|
|||
|
|
std::string modelConfigName = "model_config.json";
|
|||
|
|
_modelConfigFile = CreateFilePath(_modelFolder, modelConfigName);
|
|||
|
|
_facelivenessEngineValid = false;
|
|||
|
|
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel.zip";
|
|||
|
|
if (FileExist(livenessModelFile)) {
|
|||
|
|
_facelivenessEngineValid = InitializeLivenessModel(_licenseKey, livenessModelFile, "");
|
|||
|
|
}
|
|||
|
|
if (_facelivenessEngineValid) {
|
|||
|
|
//Initialize face tracker
|
|||
|
|
CreateANSMOTHandle(&_faceTracker, 1);
|
|||
|
|
std::string params = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
|
|||
|
|
UpdateANSMOTParameters(&_faceTracker, params.c_str());
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
//Initialize tv tracker
|
|||
|
|
CreateANSMOTHandle(&_tvTracker, 1);
|
|||
|
|
std::string paramsTv = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
|
|||
|
|
UpdateANSMOTParameters(&_tvTracker, paramsTv.c_str());
|
|||
|
|
#endif
|
|||
|
|
}
|
|||
|
|
return true; // Initialization successful
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("LoadModelFromFolder::Initialize", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
cv::Mat ANSFDBase::Preprocess(cv::Mat& input_mat, std::vector<cv::Point2f>& face_landmark_5, cv::Mat& preprocessed_mat) {
|
|||
|
|
try {
|
|||
|
|
cv::Mat crop_image;
|
|||
|
|
cv::Mat affine_martix;
|
|||
|
|
std::tie(crop_image, affine_martix) = ANSUtilityHelper::AlignFacesSCRFD(input_mat, face_landmark_5);
|
|||
|
|
return crop_image;
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::Preprocess", e.what(), __FILE__, __LINE__);
|
|||
|
|
return cv::Mat();
|
|||
|
|
}
|
|||
|
|
catch (...) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::Preprocess", "Unknown error occurred", __FILE__, __LINE__);
|
|||
|
|
return cv::Mat();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
std::string ANSFDBase::GetOpenVINODevice(ov::Core& core) {
|
|||
|
|
try {
|
|||
|
|
std::vector<std::string> available_devices = core.get_available_devices();
|
|||
|
|
bool device_found = false;
|
|||
|
|
std::string deviceName = "CPU";
|
|||
|
|
// Search for NPU
|
|||
|
|
auto it = std::find(available_devices.begin(), available_devices.end(), "NPU");
|
|||
|
|
if (it != available_devices.end()) {
|
|||
|
|
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
|
|||
|
|
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
|
|||
|
|
deviceName = "AUTO:NPU,GPU";
|
|||
|
|
device_found = true;
|
|||
|
|
return deviceName;
|
|||
|
|
}
|
|||
|
|
// If NPU not found, search for GPU
|
|||
|
|
if (!device_found) {
|
|||
|
|
it = std::find(available_devices.begin(), available_devices.end(), "GPU");
|
|||
|
|
if (it != available_devices.end()) {
|
|||
|
|
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
|
|||
|
|
deviceName = "GPU";
|
|||
|
|
device_found = true;
|
|||
|
|
return deviceName;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// If GPU not found, search for GPU.0
|
|||
|
|
if (!device_found) {
|
|||
|
|
it = std::find(available_devices.begin(), available_devices.end(), "GPU.0");
|
|||
|
|
if (it != available_devices.end()) {
|
|||
|
|
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
|
|||
|
|
deviceName = "GPU";
|
|||
|
|
device_found = true;
|
|||
|
|
return deviceName;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// If neither NPU nor GPU found, default to CPU
|
|||
|
|
if (!device_found) {
|
|||
|
|
core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
|
|||
|
|
deviceName = "CPU";
|
|||
|
|
return deviceName;
|
|||
|
|
}
|
|||
|
|
return deviceName;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::GetOpenVINODevice()", e.what(), __FILE__, __LINE__);
|
|||
|
|
core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
|
|||
|
|
return "CPU";
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
cv::Mat ANSFDBase::GetCroppedFaceScale(const cv::Mat& image,
|
|||
|
|
const int x1, const int y1, const int x2, const int y2,
|
|||
|
|
int cropedImageSize) {
|
|||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
|||
|
|
cv::Mat resizedImage;
|
|||
|
|
cv::Mat frame = image.clone();
|
|||
|
|
try {
|
|||
|
|
// Check if the input image is valid
|
|||
|
|
if (image.empty() || !image.data) return resizedImage;
|
|||
|
|
|
|||
|
|
// Calculate width and height of the face region
|
|||
|
|
int width = x2 - x1;
|
|||
|
|
int height = y2 - y1;
|
|||
|
|
|
|||
|
|
// Validate the coordinates and dimensions
|
|||
|
|
if (width <= 0 || height <= 0) return resizedImage;
|
|||
|
|
if (x1 < 0 || y1 < 0 || x2 <= x1 || y2 <= y1) return resizedImage;
|
|||
|
|
|
|||
|
|
// Define the bounding rectangle for cropping
|
|||
|
|
cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2));
|
|||
|
|
|
|||
|
|
// Ensure the rectangle has valid dimensions
|
|||
|
|
if (facePos.width <= 0 || facePos.height <= 0) return resizedImage;
|
|||
|
|
|
|||
|
|
// Apply Gaussian blur to the cropped region for smoother output
|
|||
|
|
cv::Mat tempCrop = frame(facePos).clone(); // Use clone to avoid modifying the input image
|
|||
|
|
|
|||
|
|
// Resize the cropped face to the required size
|
|||
|
|
cv::resize(tempCrop, resizedImage, cv::Size(cropedImageSize, cropedImageSize), 0, 0, cv::INTER_LANCZOS4);
|
|||
|
|
|
|||
|
|
// Release temporary resources
|
|||
|
|
tempCrop.release();
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
std::cerr << "Error in GetCroppedFaceScale: " << e.what() << std::endl;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Release the cloned frame
|
|||
|
|
frame.release();
|
|||
|
|
|
|||
|
|
return resizedImage;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
ANSFDBase::~ANSFDBase() {
|
|||
|
|
try {
|
|||
|
|
Cleanup();
|
|||
|
|
_licenseValid = false;
|
|||
|
|
_licenseKey.clear();
|
|||
|
|
if (!_cameras.empty()) // Check if _cameras has data
|
|||
|
|
{
|
|||
|
|
for (auto& [key, cameraData] : _cameras)
|
|||
|
|
{
|
|||
|
|
cameraData.clear(); // Clear each CameraData instance
|
|||
|
|
}
|
|||
|
|
_cameras.clear(); // Clear the map itself
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
std::cout << "ANSFDBase::~ANSFDBase()" << e.what();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::isSimilarObject(const Object& obj1, const Object& obj2) {
|
|||
|
|
try {
|
|||
|
|
// If both objects have valid trackIds, compare based on trackId
|
|||
|
|
if (obj1.trackId != 0 && obj2.trackId != 0) {
|
|||
|
|
return obj1.trackId == obj2.trackId;
|
|||
|
|
}
|
|||
|
|
// Compare based on classId and className
|
|||
|
|
if (obj1.classId != obj2.classId ||
|
|||
|
|
obj1.className != obj2.className ||
|
|||
|
|
obj1.cameraId != obj2.cameraId) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
// Define thresholds for similarity
|
|||
|
|
const double positionThreshold = 50.0;
|
|||
|
|
const double confidenceThreshold = 0.2;
|
|||
|
|
const double sizeThreshold = 0.2; // Allows a 20% size difference
|
|||
|
|
|
|||
|
|
// Check if centers are within positionThreshold
|
|||
|
|
cv::Point2f center1 = (obj1.box.tl() + obj1.box.br()) * 0.5;
|
|||
|
|
cv::Point2f center2 = (obj2.box.tl() + obj2.box.br()) * 0.5;
|
|||
|
|
if (cv::norm(center1 - center2) >= positionThreshold) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
// Check if bounding box sizes are similar within the size threshold
|
|||
|
|
double widthRatio = static_cast<double>(obj1.box.width) / obj2.box.width;
|
|||
|
|
double heightRatio = static_cast<double>(obj1.box.height) / obj2.box.height;
|
|||
|
|
if (std::abs(1.0 - widthRatio) > sizeThreshold || std::abs(1.0 - heightRatio) > sizeThreshold) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
// Check if confidence scores are within an acceptable range
|
|||
|
|
if (std::abs(obj1.confidence - obj2.confidence) >= confidenceThreshold) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
// If all checks pass, the objects are considered similar
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::isSimilarObject", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::isOverlayObject(const Object& obj1, const Object& obj2) {
|
|||
|
|
try {
|
|||
|
|
// If both objects have valid trackIds, compare based on trackId
|
|||
|
|
if (obj1.trackId != 0 && obj2.trackId != 0) {
|
|||
|
|
return obj1.trackId == obj2.trackId;
|
|||
|
|
}
|
|||
|
|
// Compare based on classId and className
|
|||
|
|
if (obj1.classId != obj2.classId ||
|
|||
|
|
obj1.className != obj2.className ||
|
|||
|
|
obj1.cameraId != obj2.cameraId) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Define thresholds for similarity
|
|||
|
|
const double positionThreshold = 50.0;
|
|||
|
|
|
|||
|
|
// Check if centers are within positionThreshold
|
|||
|
|
cv::Point2f center1 = (obj1.box.tl() + obj1.box.br()) * 0.5;
|
|||
|
|
cv::Point2f center2 = (obj2.box.tl() + obj2.box.br()) * 0.5;
|
|||
|
|
if (cv::norm(center1 - center2) >= positionThreshold) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Check if bounding boxes overlap
|
|||
|
|
if ((obj1.box & obj2.box).area() == 0) {
|
|||
|
|
return false; // No overlap between the bounding boxes
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// If all checks pass, the objects are considered similar
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::isOverlayObject", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
void ANSFDBase::EnqueueDetection(const std::vector<Object>& detectedObjects, const std::string& cameraId) {
|
|||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
|||
|
|
CameraData& camera = GetCameraData(cameraId);// Retrieve camera data
|
|||
|
|
if (camera._detectionQueue.size() == QUEUE_SIZE) {
|
|||
|
|
camera._detectionQueue.pop_front(); // Remove the oldest element if the queue is full
|
|||
|
|
}
|
|||
|
|
camera._detectionQueue.push_back(detectedObjects); // Add the new detection, even if empty
|
|||
|
|
}
|
|||
|
|
std::deque<std::vector<Object>>ANSFDBase::DequeueDetection(const std::string& cameraId) {
|
|||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
|||
|
|
CameraData& camera = GetCameraData(cameraId);// Retrieve camera data
|
|||
|
|
return camera._detectionQueue;
|
|||
|
|
}
|
|||
|
|
std::vector<Object> ANSFDBase::DetectMovement(const cv::Mat& input, const std::string& camera_id) {
|
|||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
|||
|
|
std::vector<Object> objects;
|
|||
|
|
objects.clear();
|
|||
|
|
try {
|
|||
|
|
if (!_licenseValid) return objects;
|
|||
|
|
if (input.empty() || !input.data || !input.u) {
|
|||
|
|
return objects;
|
|||
|
|
}
|
|||
|
|
if ((input.cols < 5) || (input.rows < 5)) return objects;
|
|||
|
|
cv::Mat frame = input.clone();
|
|||
|
|
objects = this->_handler.MovementDetect(camera_id, frame);
|
|||
|
|
frame.release();
|
|||
|
|
return objects;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::DetectMovement", e.what(), __FILE__, __LINE__);
|
|||
|
|
return objects;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
cv::Rect ANSFDBase::GenerateMinimumSquareBoundingBox(const std::vector<ANSCENTER::Object>& detectedObjects, int minSize) {
|
|||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
|||
|
|
try {
|
|||
|
|
// Ensure there are rectangles to process
|
|||
|
|
int adMinSize = minSize - 20;
|
|||
|
|
if (adMinSize < 0) adMinSize = 0;
|
|||
|
|
if (detectedObjects.empty()) return cv::Rect(0, 0, minSize, minSize);
|
|||
|
|
|
|||
|
|
// Initialize min and max coordinates
|
|||
|
|
int minX = detectedObjects[0].box.x;
|
|||
|
|
int minY = detectedObjects[0].box.y;
|
|||
|
|
int maxX = detectedObjects[0].box.x + detectedObjects[0].box.width;
|
|||
|
|
int maxY = detectedObjects[0].box.y + detectedObjects[0].box.height;
|
|||
|
|
|
|||
|
|
// Calculate bounding box that includes all rectangles
|
|||
|
|
for (const auto& rect : detectedObjects) {
|
|||
|
|
minX = std::min(minX, rect.box.x);
|
|||
|
|
minY = std::min(minY, rect.box.y);
|
|||
|
|
maxX = std::max(maxX, rect.box.x + rect.box.width);
|
|||
|
|
maxY = std::max(maxY, rect.box.y + rect.box.height);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Calculate width and height of the bounding box
|
|||
|
|
int width = maxX - minX;
|
|||
|
|
int height = maxY - minY;
|
|||
|
|
|
|||
|
|
// Determine the size of the square
|
|||
|
|
int squareSize = std::max({ width, height, adMinSize });
|
|||
|
|
|
|||
|
|
// Center the square around the bounding box
|
|||
|
|
int centerX = minX + width / 2;
|
|||
|
|
int centerY = minY + height / 2;
|
|||
|
|
int squareX = centerX - squareSize / 2;
|
|||
|
|
int squareY = centerY - squareSize / 2;
|
|||
|
|
|
|||
|
|
return cv::Rect(squareX - 10, squareY - 10, squareSize + 20, squareSize + 20); // add 10 pixels to the square
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::GenerateMinimumSquareBoundingBox", e.what(), __FILE__, __LINE__);
|
|||
|
|
return cv::Rect(0, 0, minSize, minSize);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
//bool ANSFDBase::isValidFace(const std::vector<cv::Point2f>& landmarks, const cv::Rect& faceRect, float maxEyeAngle, int offsetX, int offsetY) {
|
|||
|
|
// try {
|
|||
|
|
// // SCRFD should return exactly five landmarks in this order:
|
|||
|
|
// // [0] left eye, [1] right eye, [2] nose, [3] left mouth, [4] right mouth.
|
|||
|
|
|
|||
|
|
// int faceWidth = faceRect.width;
|
|||
|
|
// int faceHeight = faceRect.height;
|
|||
|
|
// if ((faceWidth <= 25) || (faceHeight <= 25)) return false;
|
|||
|
|
// if (landmarks.size() != 5)
|
|||
|
|
// return false;
|
|||
|
|
// std::vector<cv::Point2f> validLandmark;
|
|||
|
|
// // Ensure all landmarks lie within the face rectangle.
|
|||
|
|
// for (int i = 0; i < 5; i++)
|
|||
|
|
// {
|
|||
|
|
// cv::Point2f landmarkPoint = cv::Point2f(landmarks[i].x + offsetX, landmarks[i].y + offsetY);
|
|||
|
|
// if (!faceRect.contains(landmarkPoint))return false;
|
|||
|
|
// validLandmark.push_back(landmarkPoint);
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
// // Extract landmarks with offsets
|
|||
|
|
// cv::Point2f leftEye = validLandmark[0];
|
|||
|
|
// cv::Point2f rightEye = validLandmark[1];
|
|||
|
|
// cv::Point2f nose = validLandmark[2];
|
|||
|
|
// cv::Point2f leftMouth = validLandmark[3];
|
|||
|
|
// cv::Point2f rightMouth = validLandmark[4];
|
|||
|
|
|
|||
|
|
// // Compute the angle of the line connecting the eyes.
|
|||
|
|
// float dx = rightEye.x - leftEye.x;
|
|||
|
|
// float dy = rightEye.y - leftEye.y;
|
|||
|
|
// float eyeAngle = std::atan2(dy, dx) * 180.0f / CV_PI;
|
|||
|
|
// if (std::fabs(eyeAngle) > maxEyeAngle)
|
|||
|
|
// return false; // Face is too tilted
|
|||
|
|
|
|||
|
|
// // Compute the interocular distance.
|
|||
|
|
// float eyeDistance = std::hypot(dx, dy);
|
|||
|
|
|
|||
|
|
// // Compute mouth width.
|
|||
|
|
// float mouthWidth = cv::norm(rightMouth - leftMouth);
|
|||
|
|
|
|||
|
|
// // Check the ratio between mouth width and eye distance.
|
|||
|
|
// float ratio = mouthWidth / eyeDistance;
|
|||
|
|
// if (ratio < 0.6f || ratio > 1.6f)
|
|||
|
|
// return false;
|
|||
|
|
|
|||
|
|
// // --- Additional Nose Check ---
|
|||
|
|
|
|||
|
|
// // Compute the midpoint between the eyes.
|
|||
|
|
// cv::Point2f eyeMidpoint((leftEye.x + rightEye.x) / 2.0f, (leftEye.y + rightEye.y) / 2.0f);
|
|||
|
|
|
|||
|
|
// // Check that the nose is near the horizontal midpoint.
|
|||
|
|
// float noseDeviation = std::fabs(nose.x - eyeMidpoint.x);
|
|||
|
|
// if (noseDeviation > eyeDistance * 0.2f)
|
|||
|
|
// return false;
|
|||
|
|
|
|||
|
|
// // Check that the nose is vertically between the eyes and mouth.
|
|||
|
|
// float eyeAvgY = (leftEye.y + rightEye.y) / 2.0f;
|
|||
|
|
// float mouthAvgY = (leftMouth.y + rightMouth.y) / 2.0f;
|
|||
|
|
// if (!(nose.y > eyeAvgY && nose.y < mouthAvgY))
|
|||
|
|
// return false;
|
|||
|
|
|
|||
|
|
// return true;
|
|||
|
|
// }
|
|||
|
|
// catch (std::exception& e) {
|
|||
|
|
// this->_logger.LogFatal("ANSFDBase::isValidFace", e.what(), __FILE__, __LINE__);
|
|||
|
|
// return false;
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
//}
|
|||
|
|
bool ANSFDBase::isValidFace(const std::vector<cv::Point2f>& landmarks, const cv::Rect& faceRect,
|
|||
|
|
float maxEyeAngle, int offsetX, int offsetY, const cv::Mat& frame, float minBlurScore)
|
|||
|
|
{
|
|||
|
|
std::string rejectReason;
|
|||
|
|
|
|||
|
|
// Early validation - no exception handling needed for simple checks
|
|||
|
|
if (faceRect.width <= 25 || faceRect.height <= 25) {
|
|||
|
|
rejectReason = "too_small";
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: face too small (" << faceRect.width << "x" << faceRect.height << ")" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
if (landmarks.size() != 5) {
|
|||
|
|
rejectReason = "bad_landmarks";
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: landmark count = " << landmarks.size() << " (expected 5)" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
// SCRFD landmarks: [0] left eye, [1] right eye, [2] nose, [3] left mouth, [4] right mouth
|
|||
|
|
const float offsetXf = static_cast<float>(offsetX);
|
|||
|
|
const float offsetYf = static_cast<float>(offsetY);
|
|||
|
|
|
|||
|
|
// Apply offsets and validate all landmarks are within face rectangle
|
|||
|
|
const cv::Point2f leftEye(landmarks[0].x + offsetXf, landmarks[0].y + offsetYf);
|
|||
|
|
const cv::Point2f rightEye(landmarks[1].x + offsetXf, landmarks[1].y + offsetYf);
|
|||
|
|
const cv::Point2f nose(landmarks[2].x + offsetXf, landmarks[2].y + offsetYf);
|
|||
|
|
const cv::Point2f leftMouth(landmarks[3].x + offsetXf, landmarks[3].y + offsetYf);
|
|||
|
|
const cv::Point2f rightMouth(landmarks[4].x + offsetXf, landmarks[4].y + offsetYf);
|
|||
|
|
|
|||
|
|
// Check all landmarks are within face rectangle
|
|||
|
|
if (!faceRect.contains(leftEye) || !faceRect.contains(rightEye) ||
|
|||
|
|
!faceRect.contains(nose) || !faceRect.contains(leftMouth) ||
|
|||
|
|
!faceRect.contains(rightMouth)) {
|
|||
|
|
rejectReason = "landmarks_outside";
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: landmarks outside face rect" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
// Fall through to debug visualization below
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
{
|
|||
|
|
// Compute eye angle - reject if face is too tilted (roll check)
|
|||
|
|
const float dx = rightEye.x - leftEye.x;
|
|||
|
|
const float dy = rightEye.y - leftEye.y;
|
|||
|
|
const float eyeAngle = std::atan2(dy, dx) * (180.0f / static_cast<float>(CV_PI));
|
|||
|
|
|
|||
|
|
if (std::fabs(eyeAngle) > maxEyeAngle) {
|
|||
|
|
rejectReason = "roll=" + std::to_string(eyeAngle);
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: roll angle = " << eyeAngle << " (max " << maxEyeAngle << ")" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Compute interocular distance
|
|||
|
|
const float eyeDistance = std::hypot(dx, dy);
|
|||
|
|
if (eyeDistance < 1.0f) {
|
|||
|
|
rejectReason = "eye_dist<1";
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: interocular distance too small = " << eyeDistance << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Check mouth-to-eye ratio
|
|||
|
|
const float mouthWidth = std::hypot(rightMouth.x - leftMouth.x, rightMouth.y - leftMouth.y);
|
|||
|
|
const float ratio = mouthWidth / eyeDistance;
|
|||
|
|
|
|||
|
|
if (ratio < 0.6f || ratio > 1.6f) {
|
|||
|
|
rejectReason = "mouth_ratio=" + std::to_string(ratio);
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: mouth/eye ratio = " << ratio << " (valid 0.6-1.6)" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Yaw estimation: asymmetry ratio of nose position between eyes
|
|||
|
|
// More robust than absolute pixel deviation — scale-invariant
|
|||
|
|
const float noseToLeft = std::abs(nose.x - leftEye.x);
|
|||
|
|
const float noseToRight = std::abs(rightEye.x - nose.x);
|
|||
|
|
const float asymmetry = std::abs(noseToLeft - noseToRight) / eyeDistance;
|
|||
|
|
|
|||
|
|
if (asymmetry > 0.5f) {
|
|||
|
|
rejectReason = "yaw=" + std::to_string(asymmetry);
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: yaw asymmetry = " << asymmetry << " (max 0.5)" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Pitch estimation: nose vertical position relative to eye-mouth span
|
|||
|
|
const float eyeAvgY = (leftEye.y + rightEye.y) * 0.5f;
|
|||
|
|
const float mouthAvgY = (leftMouth.y + rightMouth.y) * 0.5f;
|
|||
|
|
const float faceHeight = mouthAvgY - eyeAvgY;
|
|||
|
|
|
|||
|
|
if (faceHeight < 1.0f) {
|
|||
|
|
rejectReason = "faceH<1";
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: degenerate face height = " << faceHeight << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const float noseRelative = (nose.y - eyeAvgY) / faceHeight;
|
|||
|
|
if (noseRelative < 0.15f || noseRelative > 0.75f) {
|
|||
|
|
rejectReason = "pitch=" + std::to_string(noseRelative);
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: pitch noseRelative = " << noseRelative << " (valid 0.15-0.75)" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Optional blur check — only when frame is provided
|
|||
|
|
if (!frame.empty()) {
|
|||
|
|
cv::Rect safeRect = faceRect & cv::Rect(0, 0, frame.cols, frame.rows);
|
|||
|
|
if (safeRect.width > 10 && safeRect.height > 10) {
|
|||
|
|
cv::Mat gray;
|
|||
|
|
cv::cvtColor(frame(safeRect), gray, cv::COLOR_BGR2GRAY);
|
|||
|
|
cv::Mat lap;
|
|||
|
|
cv::Laplacian(gray, lap, CV_64F);
|
|||
|
|
cv::Scalar mean, stddev;
|
|||
|
|
cv::meanStdDev(lap, mean, stddev);
|
|||
|
|
double blurScore = stddev.val[0] * stddev.val[0];
|
|||
|
|
if (blurScore < static_cast<double>(minBlurScore)) {
|
|||
|
|
rejectReason = "blur=" + std::to_string(blurScore);
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] REJECT: blur score = " << blurScore << " (min " << minBlurScore << ")" << std::endl;
|
|||
|
|
#endif
|
|||
|
|
goto debug_and_return;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
std::cout << "[isValidFace] PASS: roll=" << eyeAngle << " yaw=" << asymmetry
|
|||
|
|
<< " pitch=" << noseRelative << " mouthRatio=" << ratio << std::endl;
|
|||
|
|
#endif
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Debug visualization
|
|||
|
|
debug_and_return:
|
|||
|
|
#ifdef ISVALIDFACE_DEBUG
|
|||
|
|
if (!frame.empty()) {
|
|||
|
|
cv::Rect safeRect = faceRect & cv::Rect(0, 0, frame.cols, frame.rows);
|
|||
|
|
if (safeRect.width > 10 && safeRect.height > 10) {
|
|||
|
|
// Crop and scale up for visibility
|
|||
|
|
cv::Mat debugImg = frame(safeRect).clone();
|
|||
|
|
const int debugSize = 256;
|
|||
|
|
float scale = static_cast<float>(debugSize) / std::max(debugImg.cols, debugImg.rows);
|
|||
|
|
cv::resize(debugImg, debugImg, cv::Size(), scale, scale);
|
|||
|
|
|
|||
|
|
// Draw landmarks (scaled relative to face crop)
|
|||
|
|
auto toLocal = [&](const cv::Point2f& pt) -> cv::Point {
|
|||
|
|
return cv::Point(
|
|||
|
|
static_cast<int>((pt.x - safeRect.x) * scale),
|
|||
|
|
static_cast<int>((pt.y - safeRect.y) * scale));
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// Landmark colors: green=eye, blue=nose, magenta=mouth
|
|||
|
|
cv::circle(debugImg, toLocal(leftEye), 3, cv::Scalar(0, 255, 0), -1); // left eye
|
|||
|
|
cv::circle(debugImg, toLocal(rightEye), 3, cv::Scalar(0, 255, 0), -1); // right eye
|
|||
|
|
cv::circle(debugImg, toLocal(nose), 3, cv::Scalar(255, 0, 0), -1); // nose
|
|||
|
|
cv::circle(debugImg, toLocal(leftMouth), 3, cv::Scalar(255, 0, 255), -1); // left mouth
|
|||
|
|
cv::circle(debugImg, toLocal(rightMouth), 3, cv::Scalar(255, 0, 255), -1); // right mouth
|
|||
|
|
|
|||
|
|
// Draw eye line
|
|||
|
|
cv::line(debugImg, toLocal(leftEye), toLocal(rightEye), cv::Scalar(0, 200, 0), 1);
|
|||
|
|
// Draw mouth line
|
|||
|
|
cv::line(debugImg, toLocal(leftMouth), toLocal(rightMouth), cv::Scalar(200, 0, 200), 1);
|
|||
|
|
|
|||
|
|
// PASS = green border, FAIL = red border + reason
|
|||
|
|
bool passed = rejectReason.empty();
|
|||
|
|
cv::Scalar borderColor = passed ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255);
|
|||
|
|
cv::rectangle(debugImg, cv::Rect(0, 0, debugImg.cols, debugImg.rows), borderColor, 3);
|
|||
|
|
|
|||
|
|
// Draw result text
|
|||
|
|
std::string label = passed ? "PASS" : "FAIL: " + rejectReason;
|
|||
|
|
cv::putText(debugImg, label, cv::Point(5, 20),
|
|||
|
|
cv::FONT_HERSHEY_SIMPLEX, 0.5, borderColor, 1);
|
|||
|
|
|
|||
|
|
cv::imshow("isValidFace Debug", debugImg);
|
|||
|
|
cv::waitKey(1);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
#endif
|
|||
|
|
return rejectReason.empty();
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::isValidFace", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
cv::Rect ANSFDBase::computeCandidateROI(const cv::Rect& unionBox, int fixedWidth, int fixedHeight, int imageWidth, int imageHeight) {
|
|||
|
|
// Compute center of the union.
|
|||
|
|
float centerX = unionBox.x + unionBox.width / 2.0f;
|
|||
|
|
float centerY = unionBox.y + unionBox.height / 2.0f;
|
|||
|
|
|
|||
|
|
// Compute the required half-size in each dimension to cover the unionBox.
|
|||
|
|
float reqHalfWidth = std::max(centerX - static_cast<float>(unionBox.x), static_cast<float>(unionBox.x + unionBox.width) - centerX);
|
|||
|
|
float reqHalfHeight = std::max(centerY - static_cast<float>(unionBox.y), static_cast<float>(unionBox.y + unionBox.height) - centerY);
|
|||
|
|
|
|||
|
|
// Desired full dimensions: at least fixed size, but as much as needed to cover unionBox.
|
|||
|
|
int desiredWidth = static_cast<int>(std::ceil(2 * reqHalfWidth));
|
|||
|
|
int desiredHeight = static_cast<int>(std::ceil(2 * reqHalfHeight));
|
|||
|
|
|
|||
|
|
// Allow expansion up to 200% of fixed dimensions.
|
|||
|
|
int minWidth = fixedWidth;
|
|||
|
|
int minHeight = fixedHeight;
|
|||
|
|
int maxWidth = fixedWidth * 2;
|
|||
|
|
int maxHeight = fixedHeight * 2;
|
|||
|
|
|
|||
|
|
int candidateWidth = std::min(std::max(desiredWidth, minWidth), maxWidth);
|
|||
|
|
int candidateHeight = std::min(std::max(desiredHeight, minHeight), maxHeight);
|
|||
|
|
|
|||
|
|
// Compute top-left so that ROI is centered at (centerX, centerY).
|
|||
|
|
int roiX = static_cast<int>(std::round(centerX - candidateWidth / 2.0f));
|
|||
|
|
int roiY = static_cast<int>(std::round(centerY - candidateHeight / 2.0f));
|
|||
|
|
|
|||
|
|
// Clamp the ROI to image boundaries.
|
|||
|
|
if (roiX < 0) roiX = 0;
|
|||
|
|
if (roiY < 0) roiY = 0;
|
|||
|
|
if (roiX + candidateWidth > imageWidth) roiX = imageWidth - candidateWidth;
|
|||
|
|
if (roiY + candidateHeight > imageHeight) roiY = imageHeight - candidateHeight;
|
|||
|
|
|
|||
|
|
return cv::Rect(roiX, roiY, candidateWidth, candidateHeight);
|
|||
|
|
}
|
|||
|
|
std::vector<cv::Rect> ANSFDBase::GenerateFixedROIs(const std::vector<Object>& movementObjects, int fixedWidth, int fixedHeight, int imageWidth, int imageHeight) {
|
|||
|
|
try {
|
|||
|
|
// Early check: if the image is smaller than the fixed ROI size, return one ROI covering the full image.
|
|||
|
|
if (imageWidth < fixedWidth || imageHeight < fixedHeight) {
|
|||
|
|
return { cv::Rect(0, 0, imageWidth, imageHeight) };
|
|||
|
|
}
|
|||
|
|
int maxImageSize = std::max(imageWidth, imageHeight);
|
|||
|
|
if (maxImageSize <= 1920) {
|
|||
|
|
int maxFixedSize = std::max(fixedWidth, fixedHeight);
|
|||
|
|
std::vector<cv::Rect> fixedROIs;
|
|||
|
|
cv::Rect singleFixedROI = GenerateMinimumSquareBoundingBox(movementObjects, maxFixedSize);
|
|||
|
|
singleFixedROI.x = std::max(singleFixedROI.x, 0);
|
|||
|
|
singleFixedROI.y = std::max(singleFixedROI.y, 0);
|
|||
|
|
singleFixedROI.width = std::min(singleFixedROI.width, imageWidth - singleFixedROI.x);
|
|||
|
|
singleFixedROI.height = std::min(singleFixedROI.height, imageHeight - singleFixedROI.y);
|
|||
|
|
fixedROIs.push_back(singleFixedROI);
|
|||
|
|
return fixedROIs;
|
|||
|
|
}
|
|||
|
|
// --- Step 1: Group objects by proximity.
|
|||
|
|
// Use a threshold (fixedWidth/2) so that objects whose centers are within that distance belong to the same group.
|
|||
|
|
float groupingThreshold = fixedWidth / 2.0f;
|
|||
|
|
std::vector<Group> groups;
|
|||
|
|
|
|||
|
|
for (const auto& obj : movementObjects) {
|
|||
|
|
cv::Point2f center(obj.box.x + obj.box.width / 2.0f, obj.box.y + obj.box.height / 2.0f);
|
|||
|
|
bool added = false;
|
|||
|
|
for (auto& grp : groups) {
|
|||
|
|
// Use the current group's center (from its unionBox).
|
|||
|
|
cv::Point2f groupCenter(grp.unionBox.x + grp.unionBox.width / 2.0f,
|
|||
|
|
grp.unionBox.y + grp.unionBox.height / 2.0f);
|
|||
|
|
if (distance(center, groupCenter) < groupingThreshold) {
|
|||
|
|
grp.objects.push_back(obj);
|
|||
|
|
grp.unionBox = unionRect(grp.unionBox, obj.box);
|
|||
|
|
added = true;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (!added) {
|
|||
|
|
Group newGroup;
|
|||
|
|
newGroup.objects.push_back(obj);
|
|||
|
|
newGroup.unionBox = obj.box;
|
|||
|
|
groups.push_back(newGroup);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// --- Step 2: For each group, compute a candidate ROI that may expand up to 50% larger than fixed size.
|
|||
|
|
// Then merge groups whose candidate ROIs overlap.
|
|||
|
|
bool merged = true;
|
|||
|
|
while (merged) {
|
|||
|
|
merged = false;
|
|||
|
|
for (size_t i = 0; i < groups.size(); ++i) {
|
|||
|
|
cv::Rect roi_i = computeCandidateROI(groups[i].unionBox, fixedWidth, fixedHeight, imageWidth, imageHeight);
|
|||
|
|
for (size_t j = i + 1; j < groups.size(); ++j) {
|
|||
|
|
cv::Rect roi_j = computeCandidateROI(groups[j].unionBox, fixedWidth, fixedHeight, imageWidth, imageHeight);
|
|||
|
|
if (isOverlap(roi_i, roi_j)) {
|
|||
|
|
// Merge group j into group i.
|
|||
|
|
for (const auto& obj : groups[j].objects) {
|
|||
|
|
groups[i].objects.push_back(obj);
|
|||
|
|
}
|
|||
|
|
groups[i].unionBox = unionRect(groups[i].unionBox, groups[j].unionBox);
|
|||
|
|
groups.erase(groups.begin() + j);
|
|||
|
|
merged = true;
|
|||
|
|
break; // Break inner loop to restart merging.
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (merged) break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// --- Step 3: Compute final candidate ROIs for each (merged) group.
|
|||
|
|
std::vector<cv::Rect> fixedROIs;
|
|||
|
|
for (const auto& grp : groups) {
|
|||
|
|
cv::Rect candidate = computeCandidateROI(grp.unionBox, fixedWidth, fixedHeight, imageWidth, imageHeight);
|
|||
|
|
fixedROIs.push_back(candidate);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// (Optional) sort the final ROIs.
|
|||
|
|
std::sort(fixedROIs.begin(), fixedROIs.end(), [](const cv::Rect& a, const cv::Rect& b) {
|
|||
|
|
return (a.y == b.y) ? (a.x < b.x) : (a.y < b.y);
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
return fixedROIs;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::GenerateFixedROIs", e.what(), __FILE__, __LINE__);
|
|||
|
|
return { cv::Rect(0, 0, imageWidth, imageHeight) };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::ContainsIntersectingObject(const std::vector<Object>& movementObjects, const Object& result) {
|
|||
|
|
for (const auto& obj : movementObjects) {
|
|||
|
|
// Compute the intersection of the two bounding boxes.
|
|||
|
|
cv::Rect intersection = obj.box & result.box;
|
|||
|
|
if (intersection.area() > 0) {
|
|||
|
|
return true; // Found an intersecting object.
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false; // No intersections found.
|
|||
|
|
}
|
|||
|
|
void ANSFDBase::UpdateAndFilterDetectionObjects(std::vector<Object>& detectionObjects, int threshold) {
|
|||
|
|
detectionObjects.erase(
|
|||
|
|
std::remove_if(detectionObjects.begin(), detectionObjects.end(),
|
|||
|
|
[&](Object& obj) {
|
|||
|
|
if (!obj.extraInfo.empty()) {
|
|||
|
|
try {
|
|||
|
|
int value = std::stoi(obj.extraInfo); // Convert extraInfo to an integer
|
|||
|
|
if (value >= threshold) {
|
|||
|
|
return true; // Remove object if value exceeds threshold
|
|||
|
|
}
|
|||
|
|
obj.extraInfo = std::to_string(value + 1); // Increment extraInfo
|
|||
|
|
}
|
|||
|
|
catch (const std::exception&) {
|
|||
|
|
obj.extraInfo = "1"; // Reset to "1" if conversion fails
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false; // Keep object
|
|||
|
|
}),
|
|||
|
|
detectionObjects.end()); // Remove flagged objects
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool ANSFDBase::UpdateDetectionThreshold(float detectionScore) {
|
|||
|
|
try {
|
|||
|
|
this->_modelConfig.detectionScoreThreshold = detectionScore;
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::UpdateDetectionThreshold", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
float ANSFDBase::GetDetectionThreshold() {
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
return this->_modelConfig.detectionScoreThreshold;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::GetDetectionThreshold", e.what(), __FILE__, __LINE__);
|
|||
|
|
return 0.0f;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Functions for screen size division
|
|||
|
|
double ANSFDBase::calculateDistanceToCenter(const cv::Point& center, const cv::Rect& rect) {
|
|||
|
|
cv::Point rectCenter(rect.x + rect.width / 2, rect.y + rect.height / 2);
|
|||
|
|
return std::sqrt(std::pow(rectCenter.x - center.x, 2) + std::pow(rectCenter.y - center.y, 2));
|
|||
|
|
}
|
|||
|
|
std::vector<ANSFDBase::ImageSection> ANSFDBase::divideImage(const cv::Mat& image) {
|
|||
|
|
if (image.empty()) {
|
|||
|
|
std::cerr << "Error: Empty image!" << std::endl;
|
|||
|
|
return cachedSections;
|
|||
|
|
}
|
|||
|
|
cv::Size currentSize(image.cols, image.rows);
|
|||
|
|
|
|||
|
|
// Check if the image size has changed
|
|||
|
|
if (currentSize == previousImageSize) {
|
|||
|
|
return cachedSections; // Return cached sections if size is the same
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Update previous size
|
|||
|
|
previousImageSize = currentSize;
|
|||
|
|
cachedSections.clear();
|
|||
|
|
|
|||
|
|
int width = image.cols;
|
|||
|
|
int height = image.rows;
|
|||
|
|
int maxDimension = std::max(width, height);
|
|||
|
|
int numSections = 10;// std::max(1, numSections); // Ensure at least 1 section
|
|||
|
|
if (maxDimension <= 2560)numSections = 8;
|
|||
|
|
if (maxDimension <= 1280)numSections = 6;
|
|||
|
|
if (maxDimension <= 960)numSections = 4;
|
|||
|
|
if (maxDimension <= 640)numSections = 2;
|
|||
|
|
if (maxDimension <= 320)numSections = 1;
|
|||
|
|
|
|||
|
|
int gridRows = std::sqrt(numSections);
|
|||
|
|
int gridCols = (numSections + gridRows - 1) / gridRows; // Ensure all sections are covered
|
|||
|
|
|
|||
|
|
int sectionWidth = width / gridCols;
|
|||
|
|
int sectionHeight = height / gridRows;
|
|||
|
|
|
|||
|
|
cv::Point imageCenter(width / 2, height / 2);
|
|||
|
|
std::vector<std::pair<double, ImageSection>> distancePriorityList;
|
|||
|
|
|
|||
|
|
// Create sections and store their distance from the center
|
|||
|
|
for (int r = 0; r < gridRows; ++r) {
|
|||
|
|
for (int c = 0; c < gridCols; ++c) {
|
|||
|
|
int x = c * sectionWidth;
|
|||
|
|
int y = r * sectionHeight;
|
|||
|
|
int w = (c == gridCols - 1) ? width - x : sectionWidth;
|
|||
|
|
int h = (r == gridRows - 1) ? height - y : sectionHeight;
|
|||
|
|
|
|||
|
|
ImageSection section(cv::Rect(x, y, w, h));
|
|||
|
|
double distance = calculateDistanceToCenter(imageCenter, section.region);
|
|||
|
|
distancePriorityList.emplace_back(distance, section);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Sort sections based on distance from center, then top-to-bottom, then left-to-right
|
|||
|
|
std::sort(distancePriorityList.begin(), distancePriorityList.end(),
|
|||
|
|
[](const std::pair<double, ImageSection>& a, const std::pair<double, ImageSection>& b) {
|
|||
|
|
if (std::abs(a.first - b.first) > 1e-5) {
|
|||
|
|
return a.first < b.first; // Sort by closest distance to center
|
|||
|
|
}
|
|||
|
|
// If distance is the same, prioritize top to bottom, then left to right
|
|||
|
|
return a.second.region.y == b.second.region.y
|
|||
|
|
? a.second.region.x < b.second.region.x
|
|||
|
|
: a.second.region.y < b.second.region.y;
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// Assign priority
|
|||
|
|
int priority = 1;
|
|||
|
|
for (auto& entry : distancePriorityList) {
|
|||
|
|
entry.second.priority = priority++;
|
|||
|
|
cachedSections.push_back(entry.second);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return cachedSections;
|
|||
|
|
}
|
|||
|
|
std::vector<ANSFDBase::ImageSection> ANSFDBase::createSlideScreens(const cv::Mat& image) {
|
|||
|
|
if (image.empty()) {
|
|||
|
|
std::cerr << "Error: Empty image!" << std::endl;
|
|||
|
|
return cachedSections;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
cv::Size currentSize(image.cols, image.rows);
|
|||
|
|
if (currentSize == previousImageSize) {
|
|||
|
|
return cachedSections;
|
|||
|
|
}
|
|||
|
|
previousImageSize = currentSize;
|
|||
|
|
cachedSections.clear();
|
|||
|
|
|
|||
|
|
int maxSize = std::max(image.cols, image.rows);
|
|||
|
|
const int minCellSize = 320;
|
|||
|
|
int maxSections = 10;
|
|||
|
|
const float minAspectRatio = 0.8f;
|
|||
|
|
const float maxAspectRatio = 1.2f;
|
|||
|
|
|
|||
|
|
if (maxSize <= 640) maxSections = 1;
|
|||
|
|
else if (maxSize <= 960) maxSections = 2;
|
|||
|
|
else if (maxSize <= 1280) maxSections = 4;
|
|||
|
|
else if (maxSize <= 2560) maxSections = 6;
|
|||
|
|
else if (maxSize <= 3840) maxSections = 8;
|
|||
|
|
else maxSections = 10;
|
|||
|
|
|
|||
|
|
int width = image.cols;
|
|||
|
|
int height = image.rows;
|
|||
|
|
|
|||
|
|
int bestRows = 1, bestCols = 1;
|
|||
|
|
int bestTileSize = std::numeric_limits<int>::max();
|
|||
|
|
|
|||
|
|
for (int rows = 1; rows <= maxSections; ++rows) {
|
|||
|
|
for (int cols = 1; cols <= maxSections; ++cols) {
|
|||
|
|
if (rows * cols > maxSections) continue;
|
|||
|
|
|
|||
|
|
int tileWidth = (width + cols - 1) / cols;
|
|||
|
|
int tileHeight = (height + rows - 1) / rows;
|
|||
|
|
|
|||
|
|
if (tileWidth < minCellSize || tileHeight < minCellSize)
|
|||
|
|
continue;
|
|||
|
|
|
|||
|
|
float aspectRatio = static_cast<float>(tileWidth) / static_cast<float>(tileHeight);
|
|||
|
|
if (aspectRatio < minAspectRatio || aspectRatio > maxAspectRatio)
|
|||
|
|
continue;
|
|||
|
|
|
|||
|
|
int maxTileDim = std::max(tileWidth, tileHeight);
|
|||
|
|
if (maxTileDim < bestTileSize) {
|
|||
|
|
bestTileSize = maxTileDim;
|
|||
|
|
bestRows = rows;
|
|||
|
|
bestCols = cols;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Generate tiles using bestRows, bestCols
|
|||
|
|
int tileWidth = (width + bestCols - 1) / bestCols;
|
|||
|
|
int tileHeight = (height + bestRows - 1) / bestRows;
|
|||
|
|
|
|||
|
|
int priority = 1;
|
|||
|
|
for (int r = bestRows - 1; r >= 0; --r) {
|
|||
|
|
for (int c = 0; c < bestCols; ++c) {
|
|||
|
|
int x = c * tileWidth;
|
|||
|
|
int y = r * tileHeight;
|
|||
|
|
|
|||
|
|
int w = std::min(tileWidth, width - x);
|
|||
|
|
int h = std::min(tileHeight, height - y);
|
|||
|
|
|
|||
|
|
if (w <= 0 || h <= 0) continue;
|
|||
|
|
|
|||
|
|
cv::Rect region(x, y, w, h);
|
|||
|
|
ImageSection section(region);
|
|||
|
|
section.priority = priority++;
|
|||
|
|
cachedSections.push_back(section);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return cachedSections;
|
|||
|
|
}
|
|||
|
|
int ANSFDBase::getHighestPriorityRegion() {
|
|||
|
|
if (!cachedSections.empty()) {
|
|||
|
|
return cachedSections.front().priority; // First element has the highest priority
|
|||
|
|
}
|
|||
|
|
return 0; // Return empty rect if no sections exist
|
|||
|
|
}
|
|||
|
|
int ANSFDBase::getLowestPriorityRegion() {
|
|||
|
|
if (!cachedSections.empty()) {
|
|||
|
|
return cachedSections.back().priority; // Last element has the lowest priority
|
|||
|
|
}
|
|||
|
|
return 0; // Return empty rect if no sections exist
|
|||
|
|
}
|
|||
|
|
cv::Rect ANSFDBase::getRegionByPriority(int priority) {
|
|||
|
|
for (const auto& section : cachedSections) {
|
|||
|
|
if (section.priority == priority) {
|
|||
|
|
return section.region;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return cv::Rect(); // Return empty rect if priority not found
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
std::vector<Object> ANSFDBase::AdjustDetectedBoundingBoxes(
|
|||
|
|
const std::vector<Object>& detectionsInROI,
|
|||
|
|
const cv::Rect& roi,
|
|||
|
|
const cv::Size& fullImageSize,
|
|||
|
|
float aspectRatio,
|
|||
|
|
int padding
|
|||
|
|
) {
|
|||
|
|
std::vector<Object> adjustedDetections;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
// Basic input validation
|
|||
|
|
if (detectionsInROI.empty()) {
|
|||
|
|
return adjustedDetections;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (roi.width <= 0 || roi.height <= 0 ||
|
|||
|
|
fullImageSize.width <= 0 || fullImageSize.height <= 0) {
|
|||
|
|
return adjustedDetections;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (const auto& detInROI : detectionsInROI) {
|
|||
|
|
try {
|
|||
|
|
if (detInROI.box.width <= 0 || detInROI.box.height <= 0)
|
|||
|
|
continue; // Skip invalid box
|
|||
|
|
|
|||
|
|
// Convert ROI-relative box to full-image coordinates
|
|||
|
|
cv::Rect detInFullImg;
|
|||
|
|
try {
|
|||
|
|
detInFullImg = detInROI.box + roi.tl();
|
|||
|
|
detInFullImg &= cv::Rect(0, 0, fullImageSize.width, fullImageSize.height);
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
std::cerr << "[AdjustBBox] Failed to calculate full image box: " << e.what() << std::endl;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Check if it touches ROI border
|
|||
|
|
bool touchesLeft = detInROI.box.x <= 0;
|
|||
|
|
bool touchesRight = detInROI.box.x + detInROI.box.width >= roi.width - 1;
|
|||
|
|
bool touchesTop = detInROI.box.y <= 0;
|
|||
|
|
bool touchesBottom = detInROI.box.y + detInROI.box.height >= roi.height - 1;
|
|||
|
|
bool touchesBorder = touchesLeft || touchesRight || touchesTop || touchesBottom;
|
|||
|
|
|
|||
|
|
// Compute target width based on aspect ratio
|
|||
|
|
int targetWidth = 0, expandWidth = 0;
|
|||
|
|
try {
|
|||
|
|
targetWidth = std::max(detInFullImg.width, static_cast<int>(detInFullImg.height * aspectRatio));
|
|||
|
|
expandWidth = std::max(0, targetWidth - detInFullImg.width);
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
std::cerr << "[AdjustBBox] Aspect ratio adjustment failed: " << e.what() << std::endl;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
int expandLeft = expandWidth / 2;
|
|||
|
|
int expandRight = expandWidth - expandLeft;
|
|||
|
|
int padX = touchesBorder ? padding : 0;
|
|||
|
|
int padY = touchesBorder ? (padding / 2) : 0;
|
|||
|
|
|
|||
|
|
// Apply padded and expanded box
|
|||
|
|
int newX = std::max(0, detInFullImg.x - expandLeft - padX);
|
|||
|
|
int newY = std::max(0, detInFullImg.y - padY);
|
|||
|
|
int newWidth = detInFullImg.width + expandWidth + 2 * padX;
|
|||
|
|
int newHeight = detInFullImg.height + 2 * padY;
|
|||
|
|
|
|||
|
|
// Clamp to image boundaries
|
|||
|
|
if (newX + newWidth > fullImageSize.width) {
|
|||
|
|
newWidth = fullImageSize.width - newX;
|
|||
|
|
}
|
|||
|
|
if (newY + newHeight > fullImageSize.height) {
|
|||
|
|
newHeight = fullImageSize.height - newY;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (newWidth <= 0 || newHeight <= 0)
|
|||
|
|
continue;
|
|||
|
|
|
|||
|
|
// Construct adjusted object
|
|||
|
|
Object adjustedDet = detInROI;
|
|||
|
|
adjustedDet.box = cv::Rect(newX, newY, newWidth, newHeight);
|
|||
|
|
adjustedDetections.push_back(adjustedDet);
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
std::cerr << "[AdjustBBox] Exception per detection: " << e.what() << std::endl;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
catch (...) {
|
|||
|
|
std::cerr << "[AdjustBBox] Unknown exception per detection." << std::endl;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
std::cerr << "[AdjustBBox] Fatal error: " << e.what() << std::endl;
|
|||
|
|
}
|
|||
|
|
catch (...) {
|
|||
|
|
std::cerr << "[AdjustBBox] Unknown fatal error occurred." << std::endl;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return adjustedDetections;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Face detector engine implementation
|
|||
|
|
bool ANSFDBase::LoadLivenessModel(std::string antiSpoofModelPath, bool isGPU)
|
|||
|
|
{
|
|||
|
|
try {
|
|||
|
|
const auto& ep = ANSCENTER::EPLoader::Current();
|
|||
|
|
if (Ort::Global<void>::api_ == nullptr)
|
|||
|
|
Ort::InitApi(static_cast<const OrtApi*>(ANSCENTER::EPLoader::GetOrtApiRaw()));
|
|||
|
|
std::cout << "[ANSFDBase] EP ready: "<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
|
|||
|
|
|
|||
|
|
_ortLivenessEnv = new Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Liveness");
|
|||
|
|
_ortLivenessSessionOptions = new Ort::SessionOptions();
|
|||
|
|
_ortLivenessSessionOptions->SetIntraOpNumThreads(
|
|||
|
|
std::min(6, static_cast<int>(std::thread::hardware_concurrency())));
|
|||
|
|
_ortLivenessSessionOptions->SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
|
|||
|
|
|
|||
|
|
// ── Log available providers ─────────────────────────────────────────
|
|||
|
|
std::vector<std::string> availableProviders = Ort::GetAvailableProviders();
|
|||
|
|
std::cout << "[ANSFDBase] Available Execution Providers:" << std::endl;
|
|||
|
|
for (const auto& p : availableProviders)
|
|||
|
|
std::cout << " - " << p << std::endl;
|
|||
|
|
|
|||
|
|
// ── Attach EP based on runtime-detected hardware ────────────────────
|
|||
|
|
if (isGPU) {
|
|||
|
|
bool attached = false;
|
|||
|
|
|
|||
|
|
switch (ep.type) {
|
|||
|
|
|
|||
|
|
case ANSCENTER::EngineType::NVIDIA_GPU: {
|
|||
|
|
auto it = std::find(availableProviders.begin(),
|
|||
|
|
availableProviders.end(), "CUDAExecutionProvider");
|
|||
|
|
if (it == availableProviders.end()) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadLivenessModel",
|
|||
|
|
"CUDAExecutionProvider not in DLL — "
|
|||
|
|
"check ep/cuda/ has the CUDA ORT build.", __FILE__, __LINE__);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
OrtCUDAProviderOptionsV2* cuda_options = nullptr;
|
|||
|
|
Ort::GetApi().CreateCUDAProviderOptions(&cuda_options);
|
|||
|
|
|
|||
|
|
const char* keys[] = { "device_id" };
|
|||
|
|
const char* values[] = { "0" };
|
|||
|
|
Ort::GetApi().UpdateCUDAProviderOptions(cuda_options, keys, values, 1);
|
|||
|
|
|
|||
|
|
_ortLivenessSessionOptions->AppendExecutionProvider_CUDA_V2(*cuda_options);
|
|||
|
|
Ort::GetApi().ReleaseCUDAProviderOptions(cuda_options);
|
|||
|
|
|
|||
|
|
std::cout << "[ANSFDBase] CUDA EP attached." << std::endl;
|
|||
|
|
attached = true;
|
|||
|
|
}
|
|||
|
|
catch (const Ort::Exception& e) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
case ANSCENTER::EngineType::AMD_GPU: {
|
|||
|
|
auto it = std::find(availableProviders.begin(),
|
|||
|
|
availableProviders.end(), "DmlExecutionProvider");
|
|||
|
|
if (it == availableProviders.end()) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadLivenessModel",
|
|||
|
|
"DmlExecutionProvider not in DLL — "
|
|||
|
|
"check ep/directml/ has the DirectML ORT build.", __FILE__, __LINE__);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
std::unordered_map<std::string, std::string> opts = { { "device_id", "0" } };
|
|||
|
|
_ortLivenessSessionOptions->AppendExecutionProvider("DML", opts);
|
|||
|
|
std::cout << "[ANSFDBase] DirectML EP attached." << std::endl;
|
|||
|
|
attached = true;
|
|||
|
|
}
|
|||
|
|
catch (const Ort::Exception& e) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
case ANSCENTER::EngineType::OPENVINO_GPU: {
|
|||
|
|
auto it = std::find(availableProviders.begin(),
|
|||
|
|
availableProviders.end(), "OpenVINOExecutionProvider");
|
|||
|
|
if (it == availableProviders.end()) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadLivenessModel",
|
|||
|
|
"OpenVINOExecutionProvider not in DLL — "
|
|||
|
|
"check ep/openvino/ has the OpenVINO ORT build.", __FILE__, __LINE__);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const std::string precision = "FP16";
|
|||
|
|
const std::string numberOfThreads = "8";
|
|||
|
|
const std::string numberOfStreams = "8";
|
|||
|
|
|
|||
|
|
std::vector<std::unordered_map<std::string, std::string>> try_configs = {
|
|||
|
|
{ {"device_type","AUTO:NPU,GPU"}, {"precision",precision},
|
|||
|
|
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
|
|||
|
|
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
|
|||
|
|
{ {"device_type","GPU.0"}, {"precision",precision},
|
|||
|
|
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
|
|||
|
|
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
|
|||
|
|
{ {"device_type","GPU.1"}, {"precision",precision},
|
|||
|
|
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
|
|||
|
|
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
|
|||
|
|
{ {"device_type","AUTO:GPU,CPU"}, {"precision",precision},
|
|||
|
|
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
|
|||
|
|
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} }
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
for (const auto& config : try_configs) {
|
|||
|
|
try {
|
|||
|
|
_ortLivenessSessionOptions->AppendExecutionProvider_OpenVINO_V2(config);
|
|||
|
|
std::cout << "[ANSFDBase] OpenVINO EP attached ("
|
|||
|
|
<< config.at("device_type") << ")." << std::endl;
|
|||
|
|
attached = true;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
catch (const Ort::Exception& e) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!attached)
|
|||
|
|
std::cerr << "[ANSFDBase] OpenVINO EP: all device configs failed." << std::endl;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
default:
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!attached) {
|
|||
|
|
std::cerr << "[ANSFDBase] No GPU EP attached — running on CPU." << std::endl;
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::LoadLivenessModel",
|
|||
|
|
"GPU EP not attached. Running on CPU.", __FILE__, __LINE__);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
std::cout << "[ANSFDBase] Inference device: CPU (isGPU=false)" << std::endl;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── Load model ──────────────────────────────────────────────────────
|
|||
|
|
_livenessSession = new Ort::Session(
|
|||
|
|
*_ortLivenessEnv,
|
|||
|
|
std::wstring(antiSpoofModelPath.begin(), antiSpoofModelPath.end()).c_str(),
|
|||
|
|
*_ortLivenessSessionOptions);
|
|||
|
|
|
|||
|
|
// ── Node names ──────────────────────────────────────────────────────
|
|||
|
|
Ort::AllocatorWithDefaultOptions allocator;
|
|||
|
|
|
|||
|
|
Ort::AllocatedStringPtr input_name_ptr = _livenessSession->GetInputNameAllocated(0, allocator);
|
|||
|
|
_livenessInputName = input_name_ptr.get();
|
|||
|
|
|
|||
|
|
Ort::AllocatedStringPtr output_name_ptr = _livenessSession->GetOutputNameAllocated(0, allocator);
|
|||
|
|
_livenessOutputName = output_name_ptr.get();
|
|||
|
|
|
|||
|
|
std::cout << "[ANSFDBase] Liveness model loaded successfully." << std::endl;
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::InitializeLivenessModel(std::string licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) {
|
|||
|
|
try {
|
|||
|
|
_faceAttrModelFolder.clear();
|
|||
|
|
|
|||
|
|
if (!_licenseValid) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::InitializeLivenessModel", "Invalid License", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
//2. Check if model zip file exists
|
|||
|
|
if (!FileExist(modelZipFilePath)) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::InitializeLivenessModel", "Face Liveness model zip file does not exist", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//3. Unzip model file
|
|||
|
|
//_engineType = ANSLicenseHelper::CheckHardwareInformation();
|
|||
|
|
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
|
|||
|
|
std::vector<std::string> passwordArray = { modelZipPassword,
|
|||
|
|
"Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==",
|
|||
|
|
"3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==",
|
|||
|
|
"AnsCustomModels20@$",
|
|||
|
|
"AnsDemoModels20@!" };
|
|||
|
|
bool extractionSuccess = false;
|
|||
|
|
|
|||
|
|
for (const auto& password : passwordArray) {
|
|||
|
|
if (ExtractPasswordProtectedZip(modelZipFilePath, password, modelName, _faceAttrModelFolder, false)) {
|
|||
|
|
extractionSuccess = true;
|
|||
|
|
break; // Break on successful extraction
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!extractionSuccess || !std::filesystem::exists(_faceAttrModelFolder)) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::InitializeLivenessModel. Face Attribute model folder does not exist", _modelFolder, __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
_useTvDetector = false;
|
|||
|
|
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
// We can check to load TV model here (optional)
|
|||
|
|
ANSCENTER::ModelConfig _tvModelConfig;
|
|||
|
|
std::string tvLabelMap;
|
|||
|
|
std::string tvModelPath = CreateFilePath(_faceAttrModelFolder, "screen.onnx");
|
|||
|
|
if (FileExist(tvModelPath)) {
|
|||
|
|
engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation();// EngineType::CPU;//
|
|||
|
|
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
|||
|
|
// Use TensorRT YoloV11
|
|||
|
|
_tvDetector = std::make_unique <ANSCENTER::ANSYOLOV10RTOD>();
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
// Use OpenVINO YoloV11
|
|||
|
|
_tvDetector = std::make_unique <ANSCENTER::ANSOYOLOV10OVOD>();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_tvModelConfig.detectionScoreThreshold = 0.6;
|
|||
|
|
_tvModelConfig.modelConfThreshold = 0.5;
|
|||
|
|
_tvModelConfig.modelMNSThreshold = 0.5;
|
|||
|
|
|
|||
|
|
if (!_tvDetector->LoadModelFromFolder("", _tvModelConfig, "screen", "screen.names", _faceAttrModelFolder, tvLabelMap)) {
|
|||
|
|
this->_logger.LogError("ANSFDBase::InitializeLivenessModel", "Failed to load TV model from folder", __FILE__, __LINE__);
|
|||
|
|
_useTvDetector= false;
|
|||
|
|
}
|
|||
|
|
else _useTvDetector = true;
|
|||
|
|
}
|
|||
|
|
// Perform tv model optimization if needed
|
|||
|
|
std::string optimizedModelFolder;
|
|||
|
|
if (_useTvDetector && (engineType == ANSCENTER::EngineType::NVIDIA_GPU)) {
|
|||
|
|
this->_tvDetector->OptimizeModel(true, optimizedModelFolder);
|
|||
|
|
// Zip the optimized model folder
|
|||
|
|
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel_Optimized.zip";
|
|||
|
|
if (!FileExist(livenessModelFile)) {
|
|||
|
|
if (FolderExist(optimizedModelFolder)) {
|
|||
|
|
std::string modelZipPassword = "Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==";
|
|||
|
|
ZipFolderWithPassword(optimizedModelFolder.c_str(), livenessModelFile.c_str(), modelZipPassword.c_str());
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
#endif
|
|||
|
|
std::string antiSpoofModelPath = CreateFilePath(_faceAttrModelFolder, "liveness.onnx");
|
|||
|
|
if (!FileExist(antiSpoofModelPath)) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::InitializeLivenessModel", "Liveness ONNX model file does not exist", __FILE__, __LINE__);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
return LoadLivenessModel(antiSpoofModelPath, true);
|
|||
|
|
}
|
|||
|
|
catch (const std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::InitializeLivenessModel", e.what(), __FILE__, __LINE__);
|
|||
|
|
Cleanup(); // Ensure cleanup on failure
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
std::pair<int, float> ANSFDBase::PredictLiveness(const cv::Mat& image) {
|
|||
|
|
std::lock_guard<std::recursive_mutex> guard(_mutex);
|
|||
|
|
try {
|
|||
|
|
// 1. Resize
|
|||
|
|
cv::Mat faceFrame = image.clone();
|
|||
|
|
cv::resize(faceFrame, faceFrame, cv::Size(80, 80));
|
|||
|
|
|
|||
|
|
// 2. Convert to float (keep BGR order and [0,255] range)
|
|||
|
|
faceFrame.convertTo(faceFrame, CV_32F, 1.0);
|
|||
|
|
|
|||
|
|
// 3. Convert HWC to CHW
|
|||
|
|
std::vector<float> input_tensor_values(3 * 80 * 80);
|
|||
|
|
int idx = 0;
|
|||
|
|
for (int c = 0; c < 3; c++) {
|
|||
|
|
for (int y = 0; y < 80; y++) {
|
|||
|
|
for (int x = 0; x < 80; x++) {
|
|||
|
|
input_tensor_values[idx++] = faceFrame.at<cv::Vec3f>(y, x)[c];
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
std::vector<int64_t> input_shape = { 1, 3, 80, 80 };
|
|||
|
|
Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
|
|||
|
|
Ort::Value input_tensor = Ort::Value::CreateTensor<float>(
|
|||
|
|
memory_info, input_tensor_values.data(), input_tensor_values.size(),
|
|||
|
|
input_shape.data(), input_shape.size()
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
std::vector<const char*> input_names_arr{ _livenessInputName.c_str() };
|
|||
|
|
std::vector<const char*> output_names_arr{ _livenessOutputName.c_str() };
|
|||
|
|
|
|||
|
|
auto output_tensors = _livenessSession->Run(
|
|||
|
|
Ort::RunOptions{ nullptr },
|
|||
|
|
input_names_arr.data(), &input_tensor, 1,
|
|||
|
|
output_names_arr.data(), 1
|
|||
|
|
);
|
|||
|
|
auto livenessResult = LivenessPostProcessing(output_tensors.front().GetTensorMutableData<float>());
|
|||
|
|
float bliveness = livenessResult.first;
|
|||
|
|
float conf = livenessResult.second;
|
|||
|
|
faceFrame.release();
|
|||
|
|
return std::pair<int, float>{bliveness, conf};
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::PredictLiveness", e.what(), __FILE__, __LINE__);
|
|||
|
|
return std::pair<int, float>{ -1, 0.0f };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
std::pair<int, float> ANSFDBase::LivenessPostProcessing(const float* pOutput) {
|
|||
|
|
try {
|
|||
|
|
float max_val = *std::max_element(pOutput, pOutput + 3), sum = 0.0f, s[3];
|
|||
|
|
for (int i = 0; i < 3; ++i) sum += (s[i] = std::exp(pOutput[i] - max_val));
|
|||
|
|
for (int i = 0; i < 3; ++i) s[i] /= sum;
|
|||
|
|
|
|||
|
|
auto max_it = std::max_element(s, s + 3);
|
|||
|
|
int label = static_cast<int>(std::distance(s, max_it));
|
|||
|
|
float confidence = *max_it;
|
|||
|
|
|
|||
|
|
if (label != 1 && confidence > 0.6f) label = 0;
|
|||
|
|
return { label, confidence };
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::LivenessPostProcessing", e.what(), __FILE__, __LINE__);
|
|||
|
|
return std::pair<int, float>{ -1, 0.0f };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
template<typename MapTrackData, typename MapMissingFrames>
|
|||
|
|
void ANSFDBase::CleanUpTracks(std::vector<Object>& currentObjects,
|
|||
|
|
MapTrackData& trackDataMap,
|
|||
|
|
MapMissingFrames& missingFramesMap,
|
|||
|
|
const int maxMissing,
|
|||
|
|
const int maxTracks)
|
|||
|
|
{
|
|||
|
|
std::set<int> currentIds;
|
|||
|
|
for (const auto& obj : currentObjects) {
|
|||
|
|
currentIds.insert(obj.trackId);
|
|||
|
|
missingFramesMap[obj.trackId] = 0;
|
|||
|
|
}
|
|||
|
|
// Increase missing counters
|
|||
|
|
for (auto& [trackId, count] : missingFramesMap) {
|
|||
|
|
if (currentIds.find(trackId) == currentIds.end()) {
|
|||
|
|
count++;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Remove stale tracks
|
|||
|
|
for (auto it = missingFramesMap.begin(); it != missingFramesMap.end();) {
|
|||
|
|
if (it->second >= maxMissing) {
|
|||
|
|
trackDataMap.erase(it->first);
|
|||
|
|
it = missingFramesMap.erase(it);
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
++it;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Enforce max track limit
|
|||
|
|
while (trackDataMap.size() > static_cast<size_t>(maxTracks)) {
|
|||
|
|
auto it = trackDataMap.begin(); // could remove lowest ID or use heuristic
|
|||
|
|
missingFramesMap.erase(it->first);
|
|||
|
|
trackDataMap.erase(it);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
float ANSFDBase::ComputeIoU(const cv::Rect& a, const cv::Rect& b) {
|
|||
|
|
int x1 = std::max(a.x, b.x);
|
|||
|
|
int y1 = std::max(a.y, b.y);
|
|||
|
|
int x2 = std::min(a.x + a.width, b.x + b.width);
|
|||
|
|
int y2 = std::min(a.y + a.height, b.y + b.height);
|
|||
|
|
|
|||
|
|
int nInterWidth = std::max(0, x2 - x1);
|
|||
|
|
int nInterHeight = std::max(0, y2 - y1);
|
|||
|
|
int nInterArea = nInterWidth * nInterHeight;
|
|||
|
|
|
|||
|
|
int nUnionArea = a.width * a.height + b.width * b.height - nInterArea;
|
|||
|
|
|
|||
|
|
if (nUnionArea <= 0) return 0.0f;
|
|||
|
|
return static_cast<float>(nInterArea) / static_cast<float>(nUnionArea);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
std::vector<Object> ANSFDBase::TrackFaces(const cv::Mat& inputImage,const std::vector<Object> &faceObjects) {
|
|||
|
|
if (faceObjects.empty()) return faceObjects;
|
|||
|
|
if (!_facelivenessEngineValid) return faceObjects;
|
|||
|
|
std::vector<Object> vDetectedFaces;
|
|||
|
|
vDetectedFaces.reserve(faceObjects.size());
|
|||
|
|
std::lock_guard<std::recursive_mutex> guard(_mutex);
|
|||
|
|
try {
|
|||
|
|
std::vector<ANSCENTER::TrackerObject> vTrackerFace;
|
|||
|
|
//1. Prepare detected faces for tracking
|
|||
|
|
for (auto& obj : faceObjects) {
|
|||
|
|
vDetectedFaces.push_back(obj);
|
|||
|
|
// Update the tracker with the detected faces
|
|||
|
|
float pad_track_ratio = PAD_TRACK_RATIO;
|
|||
|
|
float pad_track_x = obj.box.width * pad_track_ratio;
|
|||
|
|
float pad_track_y = obj.box.height * pad_track_ratio;
|
|||
|
|
const int& class_id = obj.classId;
|
|||
|
|
const float& prob = obj.confidence;
|
|||
|
|
float x = std::max(0.0f, static_cast<float>(obj.box.x) - pad_track_x);
|
|||
|
|
float y = std::max(0.0f, static_cast<float>(obj.box.y) - pad_track_y);
|
|||
|
|
float width = std::min(obj.box.width + 2 * pad_track_x,
|
|||
|
|
static_cast<float>(inputImage.cols) - std::max(0.0f, obj.box.x - pad_track_x));
|
|||
|
|
float height = std::min(obj.box.height + 2 * pad_track_y,
|
|||
|
|
static_cast<float>(inputImage.rows) - std::max(0.0f, obj.box.y - pad_track_y));
|
|||
|
|
const float& left = x;
|
|||
|
|
const float& top = y;
|
|||
|
|
const float& right = x + width;
|
|||
|
|
const float& bottom = y + height;
|
|||
|
|
//// Create a ByteTrack::Object from the ANSCENTER::Object
|
|||
|
|
TrackerObject temp;
|
|||
|
|
temp.track_id = 0; // Placeholder, will be assigned by the tracker
|
|||
|
|
temp.prob = prob;
|
|||
|
|
temp.class_id = class_id;
|
|||
|
|
temp.x = left;
|
|||
|
|
temp.y = top;
|
|||
|
|
temp.width = width;
|
|||
|
|
temp.height = height;
|
|||
|
|
temp.left = left;
|
|||
|
|
temp.top = top;
|
|||
|
|
temp.right = right;
|
|||
|
|
temp.bottom = bottom;
|
|||
|
|
temp.object_id = obj.cameraId;
|
|||
|
|
vTrackerFace.push_back(temp);
|
|||
|
|
}
|
|||
|
|
//2. Update tracker and get tracked objects
|
|||
|
|
std::vector<TrackerObject> vObjTracker;
|
|||
|
|
UpdateANSMOTTracker(&_faceTracker, 0, vTrackerFace, vObjTracker);
|
|||
|
|
//3. Match tracker by IoU with detected face
|
|||
|
|
for (const auto& trkr : vObjTracker) {
|
|||
|
|
int nTrackId = trkr.track_id;
|
|||
|
|
cv::Rect rTrkRect(
|
|||
|
|
static_cast<int>(trkr.x),
|
|||
|
|
static_cast<int>(trkr.y),
|
|||
|
|
static_cast<int>(trkr.width),
|
|||
|
|
static_cast<int>(trkr.height)
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
int nBestIdx = -1;
|
|||
|
|
float nBestIou = 0.0f;
|
|||
|
|
|
|||
|
|
for (int i = 0; i < static_cast<int>(vDetectedFaces.size()); ++i) {
|
|||
|
|
float pad_track_ratio = PAD_TRACK_RATIO;
|
|||
|
|
int pad_x = static_cast<int>(vDetectedFaces[i].box.width * pad_track_ratio);
|
|||
|
|
int pad_y = static_cast<int>(vDetectedFaces[i].box.height * pad_track_ratio);
|
|||
|
|
|
|||
|
|
cv::Rect paddedDetectBox(
|
|||
|
|
std::max(0, vDetectedFaces[i].box.x - pad_x),
|
|||
|
|
std::max(0, vDetectedFaces[i].box.y - pad_y),
|
|||
|
|
std::min(vDetectedFaces[i].box.width + 2 * pad_x, inputImage.cols - std::max(0, vDetectedFaces[i].box.x - pad_x)),
|
|||
|
|
std::min(vDetectedFaces[i].box.height + 2 * pad_y, inputImage.rows - std::max(0, vDetectedFaces[i].box.y - pad_y))
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
float fIou = ComputeIoU(rTrkRect, paddedDetectBox);
|
|||
|
|
if (fIou > nBestIou) {
|
|||
|
|
nBestIou = fIou;
|
|||
|
|
nBestIdx = i;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Only assign trackId if IoU is above threshold
|
|||
|
|
if (nBestIdx != -1 && nBestIou > 0.5f) {
|
|||
|
|
vDetectedFaces[nBestIdx].trackId = nTrackId;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return vDetectedFaces;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::TrackFaces", e.what(), __FILE__, __LINE__);
|
|||
|
|
return vDetectedFaces;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
std::vector<Object> ANSFDBase::TrackTVScreens(const std::vector<Object>& tvObjects) {
|
|||
|
|
if (tvObjects.empty()) return tvObjects;
|
|||
|
|
if (!_facelivenessEngineValid) return tvObjects;
|
|||
|
|
std::vector<Object> vRecallScreen;
|
|||
|
|
std::lock_guard<std::recursive_mutex> guard(_mutex);
|
|||
|
|
try {
|
|||
|
|
std::vector<Object> vDetectedScreens;
|
|||
|
|
vDetectedScreens.reserve(tvObjects.size());
|
|||
|
|
std::vector<ANSCENTER::TrackerObject> vTVTracker;
|
|||
|
|
//1. Prepare detected faces for tracking
|
|||
|
|
for (auto& obj : tvObjects) {
|
|||
|
|
vDetectedScreens.push_back(obj);
|
|||
|
|
// Update the tracker with the detected faces
|
|||
|
|
const int& class_id = obj.classId;
|
|||
|
|
const float& prob = obj.confidence;
|
|||
|
|
float x = obj.box.x;
|
|||
|
|
float y = obj.box.y;
|
|||
|
|
float width = obj.box.width;
|
|||
|
|
float height = obj.box.height;
|
|||
|
|
const float& left = x;
|
|||
|
|
const float& top = y;
|
|||
|
|
const float& right = x + width;
|
|||
|
|
const float& bottom = y + height;
|
|||
|
|
//// Create a ByteTrack::Object from the ANSCENTER::Object
|
|||
|
|
TrackerObject temp;
|
|||
|
|
temp.track_id = 0; // Placeholder, will be assigned by the tracker
|
|||
|
|
temp.prob = prob;
|
|||
|
|
temp.class_id = class_id;
|
|||
|
|
temp.x = left;
|
|||
|
|
temp.y = top;
|
|||
|
|
temp.width = width;
|
|||
|
|
temp.height = height;
|
|||
|
|
temp.left = left;
|
|||
|
|
temp.top = top;
|
|||
|
|
temp.right = right;
|
|||
|
|
temp.bottom = bottom;
|
|||
|
|
temp.object_id = obj.cameraId;
|
|||
|
|
vTVTracker.push_back(temp);
|
|||
|
|
}
|
|||
|
|
//2. Update tracker and get tracked objects
|
|||
|
|
std::vector<TrackerObject> vObjTracker;
|
|||
|
|
UpdateANSMOTTracker(&_tvTracker, 0, vTVTracker, vObjTracker);
|
|||
|
|
//3. Match tracker by IoU with detected face
|
|||
|
|
for (const auto& trkr : vObjTracker) {
|
|||
|
|
int nTrackId = trkr.track_id;
|
|||
|
|
cv::Rect rTrkRect(
|
|||
|
|
static_cast<int>(trkr.x),
|
|||
|
|
static_cast<int>(trkr.y),
|
|||
|
|
static_cast<int>(trkr.width),
|
|||
|
|
static_cast<int>(trkr.height)
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
int nBestIdx = -1;
|
|||
|
|
float nBestIou = 0.0f;
|
|||
|
|
|
|||
|
|
for (int i = 0; i < static_cast<int>(vDetectedScreens.size()); ++i) {
|
|||
|
|
cv::Rect DetectBox(
|
|||
|
|
vDetectedScreens[i].box.x,
|
|||
|
|
vDetectedScreens[i].box.y,
|
|||
|
|
vDetectedScreens[i].box.width,
|
|||
|
|
vDetectedScreens[i].box.height);
|
|||
|
|
|
|||
|
|
float fIou = ComputeIoU(rTrkRect, DetectBox);
|
|||
|
|
if (fIou > nBestIou) {
|
|||
|
|
nBestIou = fIou;
|
|||
|
|
nBestIdx = i;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Only assign trackId if IoU is above threshold
|
|||
|
|
if (nBestIdx != -1 && nBestIou > 0.5f) {
|
|||
|
|
vDetectedScreens[nBestIdx].trackId = nTrackId;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (auto& screen : vDetectedScreens) {
|
|||
|
|
if (screen.trackId != -1) {
|
|||
|
|
if (screen.trackId < 0) continue;
|
|||
|
|
auto& ScreenHistory = _mTrackScreen[screen.trackId];
|
|||
|
|
ScreenHistory.push_back(screen);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
vRecallScreen.reserve(_mTrackScreen.size());
|
|||
|
|
for (auto& [id, objects] : _mTrackScreen) {
|
|||
|
|
if (!objects.empty()) {
|
|||
|
|
vRecallScreen.push_back(objects.back());
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
CleanUpTracks(vDetectedScreens, _mTrackScreen, _mMissingTrackScreen, MAX_MISSING_SCREEN, 1);
|
|||
|
|
return vRecallScreen;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::TrackTVScreens", e.what(), __FILE__, __LINE__);
|
|||
|
|
return vRecallScreen;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
bool ANSFDBase::InsideScreen(const cv::Rect& a, const cv::Rect& b) {
|
|||
|
|
if (a.x <= b.x && a.y <= b.y && a.width >= b.width && a.height >= b.height)
|
|||
|
|
return true;
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
#endif
|
|||
|
|
|
|||
|
|
std::vector<Object> ANSFDBase::ValidateLivenessFaces(const cv::Mat& inputImage,const std::vector<Object> &faceObjects, const std::string& camera_id) {
|
|||
|
|
if (!_facelivenessEngineValid) return faceObjects;
|
|||
|
|
if (faceObjects.empty()) return faceObjects;
|
|||
|
|
std::lock_guard<std::recursive_mutex> guard(_mutex);
|
|||
|
|
try {
|
|||
|
|
std::vector<Object> results;
|
|||
|
|
// 1. Track faces
|
|||
|
|
std::vector<Object> vDetectedFaces = TrackFaces(inputImage, faceObjects);
|
|||
|
|
|
|||
|
|
// 2. Run model liveness detection (vDetectedFaces with trackId assigned)
|
|||
|
|
for (auto& result : vDetectedFaces) {
|
|||
|
|
float fpad_ratio = PAD_DETECT_RATIO;
|
|||
|
|
int npad_x = static_cast<int>(result.box.width * fpad_ratio);
|
|||
|
|
int npad_y = static_cast<int>(result.box.height * fpad_ratio);
|
|||
|
|
|
|||
|
|
cv::Rect paddedROI(
|
|||
|
|
std::max(0, result.box.x - npad_x),
|
|||
|
|
std::max(0, result.box.y - npad_y),
|
|||
|
|
std::min(result.box.width + 2 * npad_x, inputImage.cols - std::max(0, result.box.x - npad_x)),
|
|||
|
|
std::min(result.box.height + 2 * npad_y, inputImage.rows - std::max(0, result.box.y - npad_y))
|
|||
|
|
);
|
|||
|
|
// Crop the face with padding
|
|||
|
|
cv::Rect faceROI = paddedROI & cv::Rect(0, 0, inputImage.cols, inputImage.rows);
|
|||
|
|
if (faceROI.width <= 0 || faceROI.height <= 0) continue;
|
|||
|
|
|
|||
|
|
cv::Mat face = inputImage(faceROI).clone();
|
|||
|
|
std::pair<int, float> prLiveness = PredictLiveness(face);
|
|||
|
|
//Return 1 if face is real, else return 0. Use for tracking attribute liveness
|
|||
|
|
int bliveness = prLiveness.first;
|
|||
|
|
|
|||
|
|
//Skip the empty face input
|
|||
|
|
if (bliveness == -1) {
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//Verify faces on screens(TV, laptop, etc.)
|
|||
|
|
bool bisFake = false;
|
|||
|
|
#ifdef USE_TV_MODEL
|
|||
|
|
if (_useTvDetector) {
|
|||
|
|
std::vector<Object> tvObjects= _tvDetector->RunInference(inputImage,camera_id); // We need to run inference here to get TV objects
|
|||
|
|
std::vector<Object> vRecallScreen = TrackTVScreens(tvObjects);
|
|||
|
|
for (const auto& obj : vRecallScreen) {
|
|||
|
|
if (InsideScreen(obj.box, result.box))
|
|||
|
|
bisFake = true;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
#endif
|
|||
|
|
// If the face (with trackId) has bliveness for MAX_HISTORY_FACE frames, we can determine real/fake
|
|||
|
|
if (result.trackId != -1) {
|
|||
|
|
auto& dqHistory = _mTrackHistory[result.trackId];
|
|||
|
|
dqHistory.push_back(prLiveness.first);
|
|||
|
|
if (dqHistory.size() >= MAX_HISTORY_FACE) {
|
|||
|
|
int sumDeque = std::accumulate(dqHistory.begin(), dqHistory.end(), 0);
|
|||
|
|
// Decide "Real" if enough real attributes and face not on Screen else "Fake"
|
|||
|
|
if (sumDeque == MAX_HISTORY_FACE && !bisFake)
|
|||
|
|
result.extraInfo = "real";
|
|||
|
|
else
|
|||
|
|
result.extraInfo = "fake";
|
|||
|
|
results.push_back(result);
|
|||
|
|
dqHistory.pop_front();
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
result.extraInfo = "unknown";
|
|||
|
|
results.push_back(result);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
CleanUpTracks(vDetectedFaces, _mTrackHistory, _mMissingTrackFrames, MAX_MISSING_FACE, MAX_TRACKS);
|
|||
|
|
|
|||
|
|
return results;
|
|||
|
|
}
|
|||
|
|
catch (std::exception& e) {
|
|||
|
|
|
|||
|
|
this->_logger.LogFatal("ANSFDBase::ValidateLivenessFaces", e.what(), __FILE__, __LINE__);
|
|||
|
|
return faceObjects;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|