Files
ANSCORE/modules/ANSODEngine/ANSFaceDetectorEngine.cpp

1953 lines
73 KiB
C++
Raw Normal View History

2026-03-28 16:54:11 +11:00
#pragma once
#include "ANSODEngine.h"
#include "ANSYOLOOD.h"
#include "ANSTENSORRTOD.h"
#include "ANSTENSORRTCL.h"
#include "ANSOPENVINOCL.h"
#include "ANSOPENVINOOD.h"
#include "ANSYOLOV10RTOD.h"
#include "ANSYOLOV10OVOD.h"
#include "ANSCUSTOMDetector.h"
#include "ANSFD.h"
#include "ANSANOMALIB.h"
#include "ANSPOSE.h"
#include "ANSSAM.h"
#include <pipelines/metadata.h>
#include <models/input_data.h>
#include "utils/visualizer.hpp"
#include <turbojpeg.h>
#include <vector>
#include <map>
#include <string>
#define USEONNXOV
//#define ISVALIDFACE_DEBUG
namespace ANSCENTER
{
void ANSFDBase::CheckLicense() {
try {
_licenseValid = ANSCENTER::ANSLicenseHelper::LicenseVerification(_licenseKey, 1004, "ANSFR");//Default productId=1004
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::CheckLicense()", e.what(), __FILE__, __LINE__);
}
}
ModelConfig ANSFDBase::GetModelConfig() {
return _modelConfig;
}
bool ANSFDBase::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)
{
Cleanup();
try {
ANSCENTER::ANSLibsLoader::Initialize();
_modelFolder = "";
_modelConfigFile = "";
_modelFolder.clear();
_modelConfigFile.clear();
// 0. Check if the modelZipFilePath exist?
if (!FileExist(modelZipFilePath)) {
this->_logger.LogFatal("ANSFDBase::LoadModel", "Model zip file is not exist", __FILE__, __LINE__);
}
// 1. Unzip model zip file to a special location with folder name as model file (and version)
std::string outputFolder;
std::vector<std::string> passwordArray;
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
passwordArray.push_back("AnsCustomModels20@$");
passwordArray.push_back("AnsDemoModels20@!");
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
//this->_logger.LogDebug("ANSFDBase::LoadModel. Model name", modelName, __FILE__, __LINE__);
size_t vectorSize = passwordArray.size();
for (size_t i = 0; i < vectorSize; i++) {
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
break; // Break the loop when the condition is met.
}
// 2. Check if the outputFolder exist
if (!std::filesystem::exists(_modelFolder)) {
this->_logger.LogError("ANSFDBase::LoadModel. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
return false; // That means the model file is not exist or the password is not correct
}
// 3. Check if the model has the configuration file
std::string modelConfigName = "model_config.json";
_modelConfigFile = CreateFilePath(_modelFolder, modelConfigName);
_facelivenessEngineValid = false;
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel.zip";
if (FileExist(livenessModelFile)) {
_facelivenessEngineValid = InitializeLivenessModel(_licenseKey, livenessModelFile, modelZipPassword);
}
if (_facelivenessEngineValid) {
//Initialize face tracker
CreateANSMOTHandle(&_faceTracker, 1);
std::string params = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
UpdateANSMOTParameters(&_faceTracker, params.c_str());
#ifdef USE_TV_MODEL
//Initialize tv tracker
CreateANSMOTHandle(&_tvTracker, 1);
std::string paramsTv = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
UpdateANSMOTParameters(&_tvTracker, paramsTv.c_str());
#endif
}
return true; // Initialization successful
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFDBase::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
ANSCENTER::ANSLibsLoader::Initialize();
#ifdef USE_TV_MODEL
if (_useTvDetector) {
if (!this->_tvDetector->OptimizeModel(fp16, optimizedModelFolder)) return false;
// Zip the optimized model folder
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel_Optimized.zip";
if (FileExist(livenessModelFile)) return true;
if (FolderExist(optimizedModelFolder)) {
std::string modelZipPassword = "Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==";
return ZipFolderWithPassword(optimizedModelFolder.c_str(), livenessModelFile.c_str(), modelZipPassword.c_str());
}
return false;
}
else return true;
#else
return true;
#endif
}
bool ANSFDBase::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
// Call Cleanup to ensure any existing resources are released before initialization
Cleanup();
try {
ANSCENTER::ANSLibsLoader::Initialize();
_licenseKey = licenseKey;
_licenseValid = false;
_modelFolder.clear();
_modelConfigFile.clear();
_imageProcessingModelFile.clear();
//1. License check
CheckLicense();
if (!_licenseValid) {
this->_logger.LogError("ANSFDBase::Initialize", "Invalid License", __FILE__, __LINE__);
return false;
}
//2. Check if model zip file exists
if (!FileExist(modelZipFilePath)) {
this->_logger.LogFatal("ANSFDBase::Initialize", "Model zip file does not exist", __FILE__, __LINE__);
return false;
}
//3. Unzip model file
//_engineType = ANSLicenseHelper::CheckHardwareInformation();
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
std::vector<std::string> passwordArray = { modelZipPassword,
"Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==",
"3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==",
"AnsCustomModels20@$",
"AnsDemoModels20@!" };
bool extractionSuccess = false;
for (const auto& password : passwordArray) {
if (ExtractPasswordProtectedZip(modelZipFilePath, password, modelName, _modelFolder, false)) {
extractionSuccess = true;
break; // Break on successful extraction
}
}
if (!extractionSuccess || !std::filesystem::exists(_modelFolder)) {
this->_logger.LogError("ANSFDBase::Initialize. Output model folder does not exist", _modelFolder, __FILE__, __LINE__);
return false;
}
_modelConfigFile = CreateFilePath(_modelFolder, "model_config.json");
// 4. Load the image processing model
//std::string imageProcessingModel = "C:\\ProgramData\\ANSCENTER\\Shared\\ANS_ImageProcessing_v1.0.zip";
//if (FileExist(imageProcessingModel)) {
// std::string imagemodelName = GetFileNameWithoutExtension(imageProcessingModel);
// std::string _imageModelFolder;
// for (const auto& password : passwordArray) {
// if (ExtractPasswordProtectedZip(imageProcessingModel, password, imagemodelName, _imageModelFolder, false)) {
// extractionSuccess = true;
// break; // Break on successful extraction
// }
// }
// _imageProcessingModelFile = CreateFilePath(_imageModelFolder, "upscale.xml");
// ov::Core core;
// if (FileExist(_imageProcessingModelFile)) {
// std::string deviceName = GetOpenVINODevice();
// try {
// if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
// {
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(28, 28), ""));
// model->setInputsPreprocessing(false, "", "");
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig(deviceName, 1, "", 4), core);
// }
// else {
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(40, 40), ""));
// model->setInputsPreprocessing(false, "", "");
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig(deviceName, 1, "", 4), core);
// }
// }
// catch (const std::exception& e) {
// // Fall back to CPU if GPU is not available
// if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
// {
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(28, 28), ""));
// model->setInputsPreprocessing(false, "", "");
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig("CPU", 1, "", 4), core);
// }
// else {
// std::unique_ptr<ImageModel> model = std::unique_ptr<ImageModel>(new SuperResolutionModel(_imageProcessingModelFile, cv::Size(40, 40), ""));
// model->setInputsPreprocessing(false, "", "");
// _pipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig("CPU", 1, "", 4), core);
// }
// }
// }
//}
// Initalize faceliveness model
_facelivenessEngineValid = false;
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel_Optimized.zip";
if (!FileExist(livenessModelFile)) livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel.zip";
if (FileExist(livenessModelFile)) {
_facelivenessEngineValid = InitializeLivenessModel(_licenseKey, livenessModelFile, modelZipPassword);
}
if (_facelivenessEngineValid) {
//Initialize face tracker
CreateANSMOTHandle(&_faceTracker, 1);
std::string params = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
UpdateANSMOTParameters(&_faceTracker, params.c_str());
#ifdef USE_TV_MODEL
//Initialize tv tracker
CreateANSMOTHandle(&_tvTracker, 1);
std::string paramsTv = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
UpdateANSMOTParameters(&_tvTracker, paramsTv.c_str());
#endif
}
return true; // Initialization successful
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSFDBase::Initialize", e.what(), __FILE__, __LINE__);
Cleanup(); // Ensure cleanup on failure
return false;
}
}
void ANSFDBase::Cleanup() {
// Clear model paths and reset related data members
_modelFolder.clear();
_modelConfigFile.clear();
_imageProcessingModelFile.clear();
if (this->_faceTracker)
{
ReleaseANSMOTHandle(&_faceTracker);
}
if (_livenessSession) {
delete _livenessSession;
}
if (_ortLivenessSessionOptions) {
delete _ortLivenessSessionOptions;
}
if (_ortLivenessEnv) {
delete _ortLivenessEnv;
}
#ifdef USE_TV_MODEL
if (this->_tvTracker)
{
ReleaseANSMOTHandle(&_tvTracker);
}
if (_useTvDetector && this->_tvDetector) {
this->_tvDetector->Destroy();
this->_tvDetector.reset();
}
#endif
}
bool ANSFDBase::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
Cleanup();
try {
ANSCENTER::ANSLibsLoader::Initialize();
_licenseKey = licenseKey;
_licenseValid = false;
_modelFolder = modelFolder;
_modelConfigFile = "";
_modelConfigFile.clear();
// 0. Check license
CheckLicense();
if (!_licenseValid) {
this->_logger.LogError("LoadModelFromFolder::Initialize", "Invalid License", __FILE__, __LINE__);
return false;
}
// 1. Check if the modelFolder exist?
if (!FolderExist(modelFolder)) {
this->_logger.LogFatal("LoadModelFromFolder::Initialize", "Model zip file is not exist", __FILE__, __LINE__);
return false;
}
// 3. Check if the model has the configuration file
std::string modelConfigName = "model_config.json";
_modelConfigFile = CreateFilePath(_modelFolder, modelConfigName);
_facelivenessEngineValid = false;
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel.zip";
if (FileExist(livenessModelFile)) {
_facelivenessEngineValid = InitializeLivenessModel(_licenseKey, livenessModelFile, "");
}
if (_facelivenessEngineValid) {
//Initialize face tracker
CreateANSMOTHandle(&_faceTracker, 1);
std::string params = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
UpdateANSMOTParameters(&_faceTracker, params.c_str());
#ifdef USE_TV_MODEL
//Initialize tv tracker
CreateANSMOTHandle(&_tvTracker, 1);
std::string paramsTv = "{\"parameters\":{\"frame_rate\":\"25\",\"track_buffer\" : \"60\",\"track_threshold\" : \"0.500000\",\"high_threshold\" : \"0.600000\",\"match_thresold\" : \"0.80000\"}}";
UpdateANSMOTParameters(&_tvTracker, paramsTv.c_str());
#endif
}
return true; // Initialization successful
}
catch (std::exception& e) {
this->_logger.LogFatal("LoadModelFromFolder::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
cv::Mat ANSFDBase::Preprocess(cv::Mat& input_mat, std::vector<cv::Point2f>& face_landmark_5, cv::Mat& preprocessed_mat) {
try {
cv::Mat crop_image;
cv::Mat affine_martix;
std::tie(crop_image, affine_martix) = ANSUtilityHelper::AlignFacesSCRFD(input_mat, face_landmark_5);
return crop_image;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSFDBase::Preprocess", e.what(), __FILE__, __LINE__);
return cv::Mat();
}
catch (...) {
this->_logger.LogFatal("ANSFDBase::Preprocess", "Unknown error occurred", __FILE__, __LINE__);
return cv::Mat();
}
}
std::string ANSFDBase::GetOpenVINODevice(ov::Core& core) {
try {
std::vector<std::string> available_devices = core.get_available_devices();
bool device_found = false;
std::string deviceName = "CPU";
// Search for NPU
auto it = std::find(available_devices.begin(), available_devices.end(), "NPU");
if (it != available_devices.end()) {
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
deviceName = "AUTO:NPU,GPU";
device_found = true;
return deviceName;
}
// If NPU not found, search for GPU
if (!device_found) {
it = std::find(available_devices.begin(), available_devices.end(), "GPU");
if (it != available_devices.end()) {
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
deviceName = "GPU";
device_found = true;
return deviceName;
}
}
// If GPU not found, search for GPU.0
if (!device_found) {
it = std::find(available_devices.begin(), available_devices.end(), "GPU.0");
if (it != available_devices.end()) {
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
deviceName = "GPU";
device_found = true;
return deviceName;
}
}
// If neither NPU nor GPU found, default to CPU
if (!device_found) {
core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
deviceName = "CPU";
return deviceName;
}
return deviceName;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::GetOpenVINODevice()", e.what(), __FILE__, __LINE__);
core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
return "CPU";
}
}
cv::Mat ANSFDBase::GetCroppedFaceScale(const cv::Mat& image,
const int x1, const int y1, const int x2, const int y2,
int cropedImageSize) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
cv::Mat resizedImage;
cv::Mat frame = image.clone();
try {
// Check if the input image is valid
if (image.empty() || !image.data) return resizedImage;
// Calculate width and height of the face region
int width = x2 - x1;
int height = y2 - y1;
// Validate the coordinates and dimensions
if (width <= 0 || height <= 0) return resizedImage;
if (x1 < 0 || y1 < 0 || x2 <= x1 || y2 <= y1) return resizedImage;
// Define the bounding rectangle for cropping
cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2));
// Ensure the rectangle has valid dimensions
if (facePos.width <= 0 || facePos.height <= 0) return resizedImage;
// Apply Gaussian blur to the cropped region for smoother output
cv::Mat tempCrop = frame(facePos).clone(); // Use clone to avoid modifying the input image
// Resize the cropped face to the required size
cv::resize(tempCrop, resizedImage, cv::Size(cropedImageSize, cropedImageSize), 0, 0, cv::INTER_LANCZOS4);
// Release temporary resources
tempCrop.release();
}
catch (const std::exception& e) {
std::cerr << "Error in GetCroppedFaceScale: " << e.what() << std::endl;
}
// Release the cloned frame
frame.release();
return resizedImage;
}
ANSFDBase::~ANSFDBase() {
try {
Cleanup();
_licenseValid = false;
_licenseKey.clear();
if (!_cameras.empty()) // Check if _cameras has data
{
for (auto& [key, cameraData] : _cameras)
{
cameraData.clear(); // Clear each CameraData instance
}
_cameras.clear(); // Clear the map itself
}
}
catch (std::exception& e) {
std::cout << "ANSFDBase::~ANSFDBase()" << e.what();
}
}
bool ANSFDBase::isSimilarObject(const Object& obj1, const Object& obj2) {
try {
// If both objects have valid trackIds, compare based on trackId
if (obj1.trackId != 0 && obj2.trackId != 0) {
return obj1.trackId == obj2.trackId;
}
// Compare based on classId and className
if (obj1.classId != obj2.classId ||
obj1.className != obj2.className ||
obj1.cameraId != obj2.cameraId) {
return false;
}
// Define thresholds for similarity
const double positionThreshold = 50.0;
const double confidenceThreshold = 0.2;
const double sizeThreshold = 0.2; // Allows a 20% size difference
// Check if centers are within positionThreshold
cv::Point2f center1 = (obj1.box.tl() + obj1.box.br()) * 0.5;
cv::Point2f center2 = (obj2.box.tl() + obj2.box.br()) * 0.5;
if (cv::norm(center1 - center2) >= positionThreshold) {
return false;
}
// Check if bounding box sizes are similar within the size threshold
double widthRatio = static_cast<double>(obj1.box.width) / obj2.box.width;
double heightRatio = static_cast<double>(obj1.box.height) / obj2.box.height;
if (std::abs(1.0 - widthRatio) > sizeThreshold || std::abs(1.0 - heightRatio) > sizeThreshold) {
return false;
}
// Check if confidence scores are within an acceptable range
if (std::abs(obj1.confidence - obj2.confidence) >= confidenceThreshold) {
return false;
}
// If all checks pass, the objects are considered similar
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::isSimilarObject", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFDBase::isOverlayObject(const Object& obj1, const Object& obj2) {
try {
// If both objects have valid trackIds, compare based on trackId
if (obj1.trackId != 0 && obj2.trackId != 0) {
return obj1.trackId == obj2.trackId;
}
// Compare based on classId and className
if (obj1.classId != obj2.classId ||
obj1.className != obj2.className ||
obj1.cameraId != obj2.cameraId) {
return false;
}
// Define thresholds for similarity
const double positionThreshold = 50.0;
// Check if centers are within positionThreshold
cv::Point2f center1 = (obj1.box.tl() + obj1.box.br()) * 0.5;
cv::Point2f center2 = (obj2.box.tl() + obj2.box.br()) * 0.5;
if (cv::norm(center1 - center2) >= positionThreshold) {
return false;
}
// Check if bounding boxes overlap
if ((obj1.box & obj2.box).area() == 0) {
return false; // No overlap between the bounding boxes
}
// If all checks pass, the objects are considered similar
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::isOverlayObject", e.what(), __FILE__, __LINE__);
return false;
}
}
void ANSFDBase::EnqueueDetection(const std::vector<Object>& detectedObjects, const std::string& cameraId) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
CameraData& camera = GetCameraData(cameraId);// Retrieve camera data
if (camera._detectionQueue.size() == QUEUE_SIZE) {
camera._detectionQueue.pop_front(); // Remove the oldest element if the queue is full
}
camera._detectionQueue.push_back(detectedObjects); // Add the new detection, even if empty
}
std::deque<std::vector<Object>>ANSFDBase::DequeueDetection(const std::string& cameraId) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
CameraData& camera = GetCameraData(cameraId);// Retrieve camera data
return camera._detectionQueue;
}
std::vector<Object> ANSFDBase::DetectMovement(const cv::Mat& input, const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<Object> objects;
objects.clear();
try {
if (!_licenseValid) return objects;
if (input.empty() || !input.data || !input.u) {
return objects;
}
if ((input.cols < 5) || (input.rows < 5)) return objects;
cv::Mat frame = input.clone();
objects = this->_handler.MovementDetect(camera_id, frame);
frame.release();
return objects;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::DetectMovement", e.what(), __FILE__, __LINE__);
return objects;
}
}
cv::Rect ANSFDBase::GenerateMinimumSquareBoundingBox(const std::vector<ANSCENTER::Object>& detectedObjects, int minSize) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Ensure there are rectangles to process
int adMinSize = minSize - 20;
if (adMinSize < 0) adMinSize = 0;
if (detectedObjects.empty()) return cv::Rect(0, 0, minSize, minSize);
// Initialize min and max coordinates
int minX = detectedObjects[0].box.x;
int minY = detectedObjects[0].box.y;
int maxX = detectedObjects[0].box.x + detectedObjects[0].box.width;
int maxY = detectedObjects[0].box.y + detectedObjects[0].box.height;
// Calculate bounding box that includes all rectangles
for (const auto& rect : detectedObjects) {
minX = std::min(minX, rect.box.x);
minY = std::min(minY, rect.box.y);
maxX = std::max(maxX, rect.box.x + rect.box.width);
maxY = std::max(maxY, rect.box.y + rect.box.height);
}
// Calculate width and height of the bounding box
int width = maxX - minX;
int height = maxY - minY;
// Determine the size of the square
int squareSize = std::max({ width, height, adMinSize });
// Center the square around the bounding box
int centerX = minX + width / 2;
int centerY = minY + height / 2;
int squareX = centerX - squareSize / 2;
int squareY = centerY - squareSize / 2;
return cv::Rect(squareX - 10, squareY - 10, squareSize + 20, squareSize + 20); // add 10 pixels to the square
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::GenerateMinimumSquareBoundingBox", e.what(), __FILE__, __LINE__);
return cv::Rect(0, 0, minSize, minSize);
}
}
//bool ANSFDBase::isValidFace(const std::vector<cv::Point2f>& landmarks, const cv::Rect& faceRect, float maxEyeAngle, int offsetX, int offsetY) {
// try {
// // SCRFD should return exactly five landmarks in this order:
// // [0] left eye, [1] right eye, [2] nose, [3] left mouth, [4] right mouth.
// int faceWidth = faceRect.width;
// int faceHeight = faceRect.height;
// if ((faceWidth <= 25) || (faceHeight <= 25)) return false;
// if (landmarks.size() != 5)
// return false;
// std::vector<cv::Point2f> validLandmark;
// // Ensure all landmarks lie within the face rectangle.
// for (int i = 0; i < 5; i++)
// {
// cv::Point2f landmarkPoint = cv::Point2f(landmarks[i].x + offsetX, landmarks[i].y + offsetY);
// if (!faceRect.contains(landmarkPoint))return false;
// validLandmark.push_back(landmarkPoint);
// }
// // Extract landmarks with offsets
// cv::Point2f leftEye = validLandmark[0];
// cv::Point2f rightEye = validLandmark[1];
// cv::Point2f nose = validLandmark[2];
// cv::Point2f leftMouth = validLandmark[3];
// cv::Point2f rightMouth = validLandmark[4];
// // Compute the angle of the line connecting the eyes.
// float dx = rightEye.x - leftEye.x;
// float dy = rightEye.y - leftEye.y;
// float eyeAngle = std::atan2(dy, dx) * 180.0f / CV_PI;
// if (std::fabs(eyeAngle) > maxEyeAngle)
// return false; // Face is too tilted
// // Compute the interocular distance.
// float eyeDistance = std::hypot(dx, dy);
// // Compute mouth width.
// float mouthWidth = cv::norm(rightMouth - leftMouth);
// // Check the ratio between mouth width and eye distance.
// float ratio = mouthWidth / eyeDistance;
// if (ratio < 0.6f || ratio > 1.6f)
// return false;
// // --- Additional Nose Check ---
// // Compute the midpoint between the eyes.
// cv::Point2f eyeMidpoint((leftEye.x + rightEye.x) / 2.0f, (leftEye.y + rightEye.y) / 2.0f);
// // Check that the nose is near the horizontal midpoint.
// float noseDeviation = std::fabs(nose.x - eyeMidpoint.x);
// if (noseDeviation > eyeDistance * 0.2f)
// return false;
// // Check that the nose is vertically between the eyes and mouth.
// float eyeAvgY = (leftEye.y + rightEye.y) / 2.0f;
// float mouthAvgY = (leftMouth.y + rightMouth.y) / 2.0f;
// if (!(nose.y > eyeAvgY && nose.y < mouthAvgY))
// return false;
// return true;
// }
// catch (std::exception& e) {
// this->_logger.LogFatal("ANSFDBase::isValidFace", e.what(), __FILE__, __LINE__);
// return false;
// }
//}
bool ANSFDBase::isValidFace(const std::vector<cv::Point2f>& landmarks, const cv::Rect& faceRect,
float maxEyeAngle, int offsetX, int offsetY, const cv::Mat& frame, float minBlurScore)
{
std::string rejectReason;
// Early validation - no exception handling needed for simple checks
if (faceRect.width <= 25 || faceRect.height <= 25) {
rejectReason = "too_small";
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: face too small (" << faceRect.width << "x" << faceRect.height << ")" << std::endl;
#endif
return false;
}
if (landmarks.size() != 5) {
rejectReason = "bad_landmarks";
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: landmark count = " << landmarks.size() << " (expected 5)" << std::endl;
#endif
return false;
}
try {
// SCRFD landmarks: [0] left eye, [1] right eye, [2] nose, [3] left mouth, [4] right mouth
const float offsetXf = static_cast<float>(offsetX);
const float offsetYf = static_cast<float>(offsetY);
// Apply offsets and validate all landmarks are within face rectangle
const cv::Point2f leftEye(landmarks[0].x + offsetXf, landmarks[0].y + offsetYf);
const cv::Point2f rightEye(landmarks[1].x + offsetXf, landmarks[1].y + offsetYf);
const cv::Point2f nose(landmarks[2].x + offsetXf, landmarks[2].y + offsetYf);
const cv::Point2f leftMouth(landmarks[3].x + offsetXf, landmarks[3].y + offsetYf);
const cv::Point2f rightMouth(landmarks[4].x + offsetXf, landmarks[4].y + offsetYf);
// Check all landmarks are within face rectangle
if (!faceRect.contains(leftEye) || !faceRect.contains(rightEye) ||
!faceRect.contains(nose) || !faceRect.contains(leftMouth) ||
!faceRect.contains(rightMouth)) {
rejectReason = "landmarks_outside";
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: landmarks outside face rect" << std::endl;
#endif
// Fall through to debug visualization below
goto debug_and_return;
}
{
// Compute eye angle - reject if face is too tilted (roll check)
const float dx = rightEye.x - leftEye.x;
const float dy = rightEye.y - leftEye.y;
const float eyeAngle = std::atan2(dy, dx) * (180.0f / static_cast<float>(CV_PI));
if (std::fabs(eyeAngle) > maxEyeAngle) {
rejectReason = "roll=" + std::to_string(eyeAngle);
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: roll angle = " << eyeAngle << " (max " << maxEyeAngle << ")" << std::endl;
#endif
goto debug_and_return;
}
// Compute interocular distance
const float eyeDistance = std::hypot(dx, dy);
if (eyeDistance < 1.0f) {
rejectReason = "eye_dist<1";
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: interocular distance too small = " << eyeDistance << std::endl;
#endif
goto debug_and_return;
}
// Check mouth-to-eye ratio
const float mouthWidth = std::hypot(rightMouth.x - leftMouth.x, rightMouth.y - leftMouth.y);
const float ratio = mouthWidth / eyeDistance;
if (ratio < 0.6f || ratio > 1.6f) {
rejectReason = "mouth_ratio=" + std::to_string(ratio);
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: mouth/eye ratio = " << ratio << " (valid 0.6-1.6)" << std::endl;
#endif
goto debug_and_return;
}
// Yaw estimation: asymmetry ratio of nose position between eyes
// More robust than absolute pixel deviation — scale-invariant
const float noseToLeft = std::abs(nose.x - leftEye.x);
const float noseToRight = std::abs(rightEye.x - nose.x);
const float asymmetry = std::abs(noseToLeft - noseToRight) / eyeDistance;
if (asymmetry > 0.5f) {
rejectReason = "yaw=" + std::to_string(asymmetry);
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: yaw asymmetry = " << asymmetry << " (max 0.5)" << std::endl;
#endif
goto debug_and_return;
}
// Pitch estimation: nose vertical position relative to eye-mouth span
const float eyeAvgY = (leftEye.y + rightEye.y) * 0.5f;
const float mouthAvgY = (leftMouth.y + rightMouth.y) * 0.5f;
const float faceHeight = mouthAvgY - eyeAvgY;
if (faceHeight < 1.0f) {
rejectReason = "faceH<1";
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: degenerate face height = " << faceHeight << std::endl;
#endif
goto debug_and_return;
}
const float noseRelative = (nose.y - eyeAvgY) / faceHeight;
if (noseRelative < 0.15f || noseRelative > 0.75f) {
rejectReason = "pitch=" + std::to_string(noseRelative);
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: pitch noseRelative = " << noseRelative << " (valid 0.15-0.75)" << std::endl;
#endif
goto debug_and_return;
}
// Optional blur check — only when frame is provided
if (!frame.empty()) {
cv::Rect safeRect = faceRect & cv::Rect(0, 0, frame.cols, frame.rows);
if (safeRect.width > 10 && safeRect.height > 10) {
cv::Mat gray;
cv::cvtColor(frame(safeRect), gray, cv::COLOR_BGR2GRAY);
cv::Mat lap;
cv::Laplacian(gray, lap, CV_64F);
cv::Scalar mean, stddev;
cv::meanStdDev(lap, mean, stddev);
double blurScore = stddev.val[0] * stddev.val[0];
if (blurScore < static_cast<double>(minBlurScore)) {
rejectReason = "blur=" + std::to_string(blurScore);
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] REJECT: blur score = " << blurScore << " (min " << minBlurScore << ")" << std::endl;
#endif
goto debug_and_return;
}
}
}
#ifdef ISVALIDFACE_DEBUG
std::cout << "[isValidFace] PASS: roll=" << eyeAngle << " yaw=" << asymmetry
<< " pitch=" << noseRelative << " mouthRatio=" << ratio << std::endl;
#endif
}
// Debug visualization
debug_and_return:
#ifdef ISVALIDFACE_DEBUG
if (!frame.empty()) {
cv::Rect safeRect = faceRect & cv::Rect(0, 0, frame.cols, frame.rows);
if (safeRect.width > 10 && safeRect.height > 10) {
// Crop and scale up for visibility
cv::Mat debugImg = frame(safeRect).clone();
const int debugSize = 256;
float scale = static_cast<float>(debugSize) / std::max(debugImg.cols, debugImg.rows);
cv::resize(debugImg, debugImg, cv::Size(), scale, scale);
// Draw landmarks (scaled relative to face crop)
auto toLocal = [&](const cv::Point2f& pt) -> cv::Point {
return cv::Point(
static_cast<int>((pt.x - safeRect.x) * scale),
static_cast<int>((pt.y - safeRect.y) * scale));
};
// Landmark colors: green=eye, blue=nose, magenta=mouth
cv::circle(debugImg, toLocal(leftEye), 3, cv::Scalar(0, 255, 0), -1); // left eye
cv::circle(debugImg, toLocal(rightEye), 3, cv::Scalar(0, 255, 0), -1); // right eye
cv::circle(debugImg, toLocal(nose), 3, cv::Scalar(255, 0, 0), -1); // nose
cv::circle(debugImg, toLocal(leftMouth), 3, cv::Scalar(255, 0, 255), -1); // left mouth
cv::circle(debugImg, toLocal(rightMouth), 3, cv::Scalar(255, 0, 255), -1); // right mouth
// Draw eye line
cv::line(debugImg, toLocal(leftEye), toLocal(rightEye), cv::Scalar(0, 200, 0), 1);
// Draw mouth line
cv::line(debugImg, toLocal(leftMouth), toLocal(rightMouth), cv::Scalar(200, 0, 200), 1);
// PASS = green border, FAIL = red border + reason
bool passed = rejectReason.empty();
cv::Scalar borderColor = passed ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255);
cv::rectangle(debugImg, cv::Rect(0, 0, debugImg.cols, debugImg.rows), borderColor, 3);
// Draw result text
std::string label = passed ? "PASS" : "FAIL: " + rejectReason;
cv::putText(debugImg, label, cv::Point(5, 20),
cv::FONT_HERSHEY_SIMPLEX, 0.5, borderColor, 1);
cv::imshow("isValidFace Debug", debugImg);
cv::waitKey(1);
}
}
#endif
return rejectReason.empty();
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSFDBase::isValidFace", e.what(), __FILE__, __LINE__);
return false;
}
}
cv::Rect ANSFDBase::computeCandidateROI(const cv::Rect& unionBox, int fixedWidth, int fixedHeight, int imageWidth, int imageHeight) {
// Compute center of the union.
float centerX = unionBox.x + unionBox.width / 2.0f;
float centerY = unionBox.y + unionBox.height / 2.0f;
// Compute the required half-size in each dimension to cover the unionBox.
float reqHalfWidth = std::max(centerX - static_cast<float>(unionBox.x), static_cast<float>(unionBox.x + unionBox.width) - centerX);
float reqHalfHeight = std::max(centerY - static_cast<float>(unionBox.y), static_cast<float>(unionBox.y + unionBox.height) - centerY);
// Desired full dimensions: at least fixed size, but as much as needed to cover unionBox.
int desiredWidth = static_cast<int>(std::ceil(2 * reqHalfWidth));
int desiredHeight = static_cast<int>(std::ceil(2 * reqHalfHeight));
// Allow expansion up to 200% of fixed dimensions.
int minWidth = fixedWidth;
int minHeight = fixedHeight;
int maxWidth = fixedWidth * 2;
int maxHeight = fixedHeight * 2;
int candidateWidth = std::min(std::max(desiredWidth, minWidth), maxWidth);
int candidateHeight = std::min(std::max(desiredHeight, minHeight), maxHeight);
// Compute top-left so that ROI is centered at (centerX, centerY).
int roiX = static_cast<int>(std::round(centerX - candidateWidth / 2.0f));
int roiY = static_cast<int>(std::round(centerY - candidateHeight / 2.0f));
// Clamp the ROI to image boundaries.
if (roiX < 0) roiX = 0;
if (roiY < 0) roiY = 0;
if (roiX + candidateWidth > imageWidth) roiX = imageWidth - candidateWidth;
if (roiY + candidateHeight > imageHeight) roiY = imageHeight - candidateHeight;
return cv::Rect(roiX, roiY, candidateWidth, candidateHeight);
}
std::vector<cv::Rect> ANSFDBase::GenerateFixedROIs(const std::vector<Object>& movementObjects, int fixedWidth, int fixedHeight, int imageWidth, int imageHeight) {
try {
// Early check: if the image is smaller than the fixed ROI size, return one ROI covering the full image.
if (imageWidth < fixedWidth || imageHeight < fixedHeight) {
return { cv::Rect(0, 0, imageWidth, imageHeight) };
}
int maxImageSize = std::max(imageWidth, imageHeight);
if (maxImageSize <= 1920) {
int maxFixedSize = std::max(fixedWidth, fixedHeight);
std::vector<cv::Rect> fixedROIs;
cv::Rect singleFixedROI = GenerateMinimumSquareBoundingBox(movementObjects, maxFixedSize);
singleFixedROI.x = std::max(singleFixedROI.x, 0);
singleFixedROI.y = std::max(singleFixedROI.y, 0);
singleFixedROI.width = std::min(singleFixedROI.width, imageWidth - singleFixedROI.x);
singleFixedROI.height = std::min(singleFixedROI.height, imageHeight - singleFixedROI.y);
fixedROIs.push_back(singleFixedROI);
return fixedROIs;
}
// --- Step 1: Group objects by proximity.
// Use a threshold (fixedWidth/2) so that objects whose centers are within that distance belong to the same group.
float groupingThreshold = fixedWidth / 2.0f;
std::vector<Group> groups;
for (const auto& obj : movementObjects) {
cv::Point2f center(obj.box.x + obj.box.width / 2.0f, obj.box.y + obj.box.height / 2.0f);
bool added = false;
for (auto& grp : groups) {
// Use the current group's center (from its unionBox).
cv::Point2f groupCenter(grp.unionBox.x + grp.unionBox.width / 2.0f,
grp.unionBox.y + grp.unionBox.height / 2.0f);
if (distance(center, groupCenter) < groupingThreshold) {
grp.objects.push_back(obj);
grp.unionBox = unionRect(grp.unionBox, obj.box);
added = true;
break;
}
}
if (!added) {
Group newGroup;
newGroup.objects.push_back(obj);
newGroup.unionBox = obj.box;
groups.push_back(newGroup);
}
}
// --- Step 2: For each group, compute a candidate ROI that may expand up to 50% larger than fixed size.
// Then merge groups whose candidate ROIs overlap.
bool merged = true;
while (merged) {
merged = false;
for (size_t i = 0; i < groups.size(); ++i) {
cv::Rect roi_i = computeCandidateROI(groups[i].unionBox, fixedWidth, fixedHeight, imageWidth, imageHeight);
for (size_t j = i + 1; j < groups.size(); ++j) {
cv::Rect roi_j = computeCandidateROI(groups[j].unionBox, fixedWidth, fixedHeight, imageWidth, imageHeight);
if (isOverlap(roi_i, roi_j)) {
// Merge group j into group i.
for (const auto& obj : groups[j].objects) {
groups[i].objects.push_back(obj);
}
groups[i].unionBox = unionRect(groups[i].unionBox, groups[j].unionBox);
groups.erase(groups.begin() + j);
merged = true;
break; // Break inner loop to restart merging.
}
}
if (merged) break;
}
}
// --- Step 3: Compute final candidate ROIs for each (merged) group.
std::vector<cv::Rect> fixedROIs;
for (const auto& grp : groups) {
cv::Rect candidate = computeCandidateROI(grp.unionBox, fixedWidth, fixedHeight, imageWidth, imageHeight);
fixedROIs.push_back(candidate);
}
// (Optional) sort the final ROIs.
std::sort(fixedROIs.begin(), fixedROIs.end(), [](const cv::Rect& a, const cv::Rect& b) {
return (a.y == b.y) ? (a.x < b.x) : (a.y < b.y);
});
return fixedROIs;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::GenerateFixedROIs", e.what(), __FILE__, __LINE__);
return { cv::Rect(0, 0, imageWidth, imageHeight) };
}
}
bool ANSFDBase::ContainsIntersectingObject(const std::vector<Object>& movementObjects, const Object& result) {
for (const auto& obj : movementObjects) {
// Compute the intersection of the two bounding boxes.
cv::Rect intersection = obj.box & result.box;
if (intersection.area() > 0) {
return true; // Found an intersecting object.
}
}
return false; // No intersections found.
}
void ANSFDBase::UpdateAndFilterDetectionObjects(std::vector<Object>& detectionObjects, int threshold) {
detectionObjects.erase(
std::remove_if(detectionObjects.begin(), detectionObjects.end(),
[&](Object& obj) {
if (!obj.extraInfo.empty()) {
try {
int value = std::stoi(obj.extraInfo); // Convert extraInfo to an integer
if (value >= threshold) {
return true; // Remove object if value exceeds threshold
}
obj.extraInfo = std::to_string(value + 1); // Increment extraInfo
}
catch (const std::exception&) {
obj.extraInfo = "1"; // Reset to "1" if conversion fails
}
}
return false; // Keep object
}),
detectionObjects.end()); // Remove flagged objects
}
bool ANSFDBase::UpdateDetectionThreshold(float detectionScore) {
try {
this->_modelConfig.detectionScoreThreshold = detectionScore;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::UpdateDetectionThreshold", e.what(), __FILE__, __LINE__);
return false;
}
}
float ANSFDBase::GetDetectionThreshold() {
try {
return this->_modelConfig.detectionScoreThreshold;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::GetDetectionThreshold", e.what(), __FILE__, __LINE__);
return 0.0f;
}
}
// Functions for screen size division
double ANSFDBase::calculateDistanceToCenter(const cv::Point& center, const cv::Rect& rect) {
cv::Point rectCenter(rect.x + rect.width / 2, rect.y + rect.height / 2);
return std::sqrt(std::pow(rectCenter.x - center.x, 2) + std::pow(rectCenter.y - center.y, 2));
}
std::vector<ANSFDBase::ImageSection> ANSFDBase::divideImage(const cv::Mat& image) {
if (image.empty()) {
std::cerr << "Error: Empty image!" << std::endl;
return cachedSections;
}
cv::Size currentSize(image.cols, image.rows);
// Check if the image size has changed
if (currentSize == previousImageSize) {
return cachedSections; // Return cached sections if size is the same
}
// Update previous size
previousImageSize = currentSize;
cachedSections.clear();
int width = image.cols;
int height = image.rows;
int maxDimension = std::max(width, height);
int numSections = 10;// std::max(1, numSections); // Ensure at least 1 section
if (maxDimension <= 2560)numSections = 8;
if (maxDimension <= 1280)numSections = 6;
if (maxDimension <= 960)numSections = 4;
if (maxDimension <= 640)numSections = 2;
if (maxDimension <= 320)numSections = 1;
int gridRows = std::sqrt(numSections);
int gridCols = (numSections + gridRows - 1) / gridRows; // Ensure all sections are covered
int sectionWidth = width / gridCols;
int sectionHeight = height / gridRows;
cv::Point imageCenter(width / 2, height / 2);
std::vector<std::pair<double, ImageSection>> distancePriorityList;
// Create sections and store their distance from the center
for (int r = 0; r < gridRows; ++r) {
for (int c = 0; c < gridCols; ++c) {
int x = c * sectionWidth;
int y = r * sectionHeight;
int w = (c == gridCols - 1) ? width - x : sectionWidth;
int h = (r == gridRows - 1) ? height - y : sectionHeight;
ImageSection section(cv::Rect(x, y, w, h));
double distance = calculateDistanceToCenter(imageCenter, section.region);
distancePriorityList.emplace_back(distance, section);
}
}
// Sort sections based on distance from center, then top-to-bottom, then left-to-right
std::sort(distancePriorityList.begin(), distancePriorityList.end(),
[](const std::pair<double, ImageSection>& a, const std::pair<double, ImageSection>& b) {
if (std::abs(a.first - b.first) > 1e-5) {
return a.first < b.first; // Sort by closest distance to center
}
// If distance is the same, prioritize top to bottom, then left to right
return a.second.region.y == b.second.region.y
? a.second.region.x < b.second.region.x
: a.second.region.y < b.second.region.y;
});
// Assign priority
int priority = 1;
for (auto& entry : distancePriorityList) {
entry.second.priority = priority++;
cachedSections.push_back(entry.second);
}
return cachedSections;
}
std::vector<ANSFDBase::ImageSection> ANSFDBase::createSlideScreens(const cv::Mat& image) {
if (image.empty()) {
std::cerr << "Error: Empty image!" << std::endl;
return cachedSections;
}
cv::Size currentSize(image.cols, image.rows);
if (currentSize == previousImageSize) {
return cachedSections;
}
previousImageSize = currentSize;
cachedSections.clear();
int maxSize = std::max(image.cols, image.rows);
const int minCellSize = 320;
int maxSections = 10;
const float minAspectRatio = 0.8f;
const float maxAspectRatio = 1.2f;
if (maxSize <= 640) maxSections = 1;
else if (maxSize <= 960) maxSections = 2;
else if (maxSize <= 1280) maxSections = 4;
else if (maxSize <= 2560) maxSections = 6;
else if (maxSize <= 3840) maxSections = 8;
else maxSections = 10;
int width = image.cols;
int height = image.rows;
int bestRows = 1, bestCols = 1;
int bestTileSize = std::numeric_limits<int>::max();
for (int rows = 1; rows <= maxSections; ++rows) {
for (int cols = 1; cols <= maxSections; ++cols) {
if (rows * cols > maxSections) continue;
int tileWidth = (width + cols - 1) / cols;
int tileHeight = (height + rows - 1) / rows;
if (tileWidth < minCellSize || tileHeight < minCellSize)
continue;
float aspectRatio = static_cast<float>(tileWidth) / static_cast<float>(tileHeight);
if (aspectRatio < minAspectRatio || aspectRatio > maxAspectRatio)
continue;
int maxTileDim = std::max(tileWidth, tileHeight);
if (maxTileDim < bestTileSize) {
bestTileSize = maxTileDim;
bestRows = rows;
bestCols = cols;
}
}
}
// Generate tiles using bestRows, bestCols
int tileWidth = (width + bestCols - 1) / bestCols;
int tileHeight = (height + bestRows - 1) / bestRows;
int priority = 1;
for (int r = bestRows - 1; r >= 0; --r) {
for (int c = 0; c < bestCols; ++c) {
int x = c * tileWidth;
int y = r * tileHeight;
int w = std::min(tileWidth, width - x);
int h = std::min(tileHeight, height - y);
if (w <= 0 || h <= 0) continue;
cv::Rect region(x, y, w, h);
ImageSection section(region);
section.priority = priority++;
cachedSections.push_back(section);
}
}
return cachedSections;
}
int ANSFDBase::getHighestPriorityRegion() {
if (!cachedSections.empty()) {
return cachedSections.front().priority; // First element has the highest priority
}
return 0; // Return empty rect if no sections exist
}
int ANSFDBase::getLowestPriorityRegion() {
if (!cachedSections.empty()) {
return cachedSections.back().priority; // Last element has the lowest priority
}
return 0; // Return empty rect if no sections exist
}
cv::Rect ANSFDBase::getRegionByPriority(int priority) {
for (const auto& section : cachedSections) {
if (section.priority == priority) {
return section.region;
}
}
return cv::Rect(); // Return empty rect if priority not found
}
std::vector<Object> ANSFDBase::AdjustDetectedBoundingBoxes(
const std::vector<Object>& detectionsInROI,
const cv::Rect& roi,
const cv::Size& fullImageSize,
float aspectRatio,
int padding
) {
std::vector<Object> adjustedDetections;
try {
// Basic input validation
if (detectionsInROI.empty()) {
return adjustedDetections;
}
if (roi.width <= 0 || roi.height <= 0 ||
fullImageSize.width <= 0 || fullImageSize.height <= 0) {
return adjustedDetections;
}
for (const auto& detInROI : detectionsInROI) {
try {
if (detInROI.box.width <= 0 || detInROI.box.height <= 0)
continue; // Skip invalid box
// Convert ROI-relative box to full-image coordinates
cv::Rect detInFullImg;
try {
detInFullImg = detInROI.box + roi.tl();
detInFullImg &= cv::Rect(0, 0, fullImageSize.width, fullImageSize.height);
}
catch (const std::exception& e) {
std::cerr << "[AdjustBBox] Failed to calculate full image box: " << e.what() << std::endl;
continue;
}
// Check if it touches ROI border
bool touchesLeft = detInROI.box.x <= 0;
bool touchesRight = detInROI.box.x + detInROI.box.width >= roi.width - 1;
bool touchesTop = detInROI.box.y <= 0;
bool touchesBottom = detInROI.box.y + detInROI.box.height >= roi.height - 1;
bool touchesBorder = touchesLeft || touchesRight || touchesTop || touchesBottom;
// Compute target width based on aspect ratio
int targetWidth = 0, expandWidth = 0;
try {
targetWidth = std::max(detInFullImg.width, static_cast<int>(detInFullImg.height * aspectRatio));
expandWidth = std::max(0, targetWidth - detInFullImg.width);
}
catch (const std::exception& e) {
std::cerr << "[AdjustBBox] Aspect ratio adjustment failed: " << e.what() << std::endl;
continue;
}
int expandLeft = expandWidth / 2;
int expandRight = expandWidth - expandLeft;
int padX = touchesBorder ? padding : 0;
int padY = touchesBorder ? (padding / 2) : 0;
// Apply padded and expanded box
int newX = std::max(0, detInFullImg.x - expandLeft - padX);
int newY = std::max(0, detInFullImg.y - padY);
int newWidth = detInFullImg.width + expandWidth + 2 * padX;
int newHeight = detInFullImg.height + 2 * padY;
// Clamp to image boundaries
if (newX + newWidth > fullImageSize.width) {
newWidth = fullImageSize.width - newX;
}
if (newY + newHeight > fullImageSize.height) {
newHeight = fullImageSize.height - newY;
}
if (newWidth <= 0 || newHeight <= 0)
continue;
// Construct adjusted object
Object adjustedDet = detInROI;
adjustedDet.box = cv::Rect(newX, newY, newWidth, newHeight);
adjustedDetections.push_back(adjustedDet);
}
catch (const std::exception& e) {
std::cerr << "[AdjustBBox] Exception per detection: " << e.what() << std::endl;
continue;
}
catch (...) {
std::cerr << "[AdjustBBox] Unknown exception per detection." << std::endl;
continue;
}
}
}
catch (const std::exception& e) {
std::cerr << "[AdjustBBox] Fatal error: " << e.what() << std::endl;
}
catch (...) {
std::cerr << "[AdjustBBox] Unknown fatal error occurred." << std::endl;
}
return adjustedDetections;
}
// Face detector engine implementation
bool ANSFDBase::LoadLivenessModel(std::string antiSpoofModelPath, bool isGPU)
{
try {
const auto& ep = ANSCENTER::EPLoader::Current();
if (Ort::Global<void>::api_ == nullptr)
Ort::InitApi(static_cast<const OrtApi*>(ANSCENTER::EPLoader::GetOrtApiRaw()));
std::cout << "[ANSFDBase] EP ready: "<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
_ortLivenessEnv = new Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Liveness");
_ortLivenessSessionOptions = new Ort::SessionOptions();
_ortLivenessSessionOptions->SetIntraOpNumThreads(
std::min(6, static_cast<int>(std::thread::hardware_concurrency())));
_ortLivenessSessionOptions->SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
// ── Log available providers ─────────────────────────────────────────
std::vector<std::string> availableProviders = Ort::GetAvailableProviders();
std::cout << "[ANSFDBase] Available Execution Providers:" << std::endl;
for (const auto& p : availableProviders)
std::cout << " - " << p << std::endl;
// ── Attach EP based on runtime-detected hardware ────────────────────
if (isGPU) {
bool attached = false;
switch (ep.type) {
case ANSCENTER::EngineType::NVIDIA_GPU: {
auto it = std::find(availableProviders.begin(),
availableProviders.end(), "CUDAExecutionProvider");
if (it == availableProviders.end()) {
this->_logger.LogError("ANSFDBase::LoadLivenessModel",
"CUDAExecutionProvider not in DLL — "
"check ep/cuda/ has the CUDA ORT build.", __FILE__, __LINE__);
break;
}
try {
OrtCUDAProviderOptionsV2* cuda_options = nullptr;
Ort::GetApi().CreateCUDAProviderOptions(&cuda_options);
const char* keys[] = { "device_id" };
const char* values[] = { "0" };
Ort::GetApi().UpdateCUDAProviderOptions(cuda_options, keys, values, 1);
_ortLivenessSessionOptions->AppendExecutionProvider_CUDA_V2(*cuda_options);
Ort::GetApi().ReleaseCUDAProviderOptions(cuda_options);
std::cout << "[ANSFDBase] CUDA EP attached." << std::endl;
attached = true;
}
catch (const Ort::Exception& e) {
this->_logger.LogError("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
}
break;
}
case ANSCENTER::EngineType::AMD_GPU: {
auto it = std::find(availableProviders.begin(),
availableProviders.end(), "DmlExecutionProvider");
if (it == availableProviders.end()) {
this->_logger.LogError("ANSFDBase::LoadLivenessModel",
"DmlExecutionProvider not in DLL — "
"check ep/directml/ has the DirectML ORT build.", __FILE__, __LINE__);
break;
}
try {
std::unordered_map<std::string, std::string> opts = { { "device_id", "0" } };
_ortLivenessSessionOptions->AppendExecutionProvider("DML", opts);
std::cout << "[ANSFDBase] DirectML EP attached." << std::endl;
attached = true;
}
catch (const Ort::Exception& e) {
this->_logger.LogError("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
}
break;
}
case ANSCENTER::EngineType::OPENVINO_GPU: {
auto it = std::find(availableProviders.begin(),
availableProviders.end(), "OpenVINOExecutionProvider");
if (it == availableProviders.end()) {
this->_logger.LogError("ANSFDBase::LoadLivenessModel",
"OpenVINOExecutionProvider not in DLL — "
"check ep/openvino/ has the OpenVINO ORT build.", __FILE__, __LINE__);
break;
}
const std::string precision = "FP16";
const std::string numberOfThreads = "8";
const std::string numberOfStreams = "8";
std::vector<std::unordered_map<std::string, std::string>> try_configs = {
{ {"device_type","AUTO:NPU,GPU"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
{ {"device_type","GPU.0"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
{ {"device_type","GPU.1"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
{ {"device_type","AUTO:GPU,CPU"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} }
};
for (const auto& config : try_configs) {
try {
_ortLivenessSessionOptions->AppendExecutionProvider_OpenVINO_V2(config);
std::cout << "[ANSFDBase] OpenVINO EP attached ("
<< config.at("device_type") << ")." << std::endl;
attached = true;
break;
}
catch (const Ort::Exception& e) {
this->_logger.LogError("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
}
}
if (!attached)
std::cerr << "[ANSFDBase] OpenVINO EP: all device configs failed." << std::endl;
break;
}
default:
break;
}
if (!attached) {
std::cerr << "[ANSFDBase] No GPU EP attached — running on CPU." << std::endl;
this->_logger.LogFatal("ANSFDBase::LoadLivenessModel",
"GPU EP not attached. Running on CPU.", __FILE__, __LINE__);
}
}
else {
std::cout << "[ANSFDBase] Inference device: CPU (isGPU=false)" << std::endl;
}
// ── Load model ──────────────────────────────────────────────────────
_livenessSession = new Ort::Session(
*_ortLivenessEnv,
std::wstring(antiSpoofModelPath.begin(), antiSpoofModelPath.end()).c_str(),
*_ortLivenessSessionOptions);
// ── Node names ──────────────────────────────────────────────────────
Ort::AllocatorWithDefaultOptions allocator;
Ort::AllocatedStringPtr input_name_ptr = _livenessSession->GetInputNameAllocated(0, allocator);
_livenessInputName = input_name_ptr.get();
Ort::AllocatedStringPtr output_name_ptr = _livenessSession->GetOutputNameAllocated(0, allocator);
_livenessOutputName = output_name_ptr.get();
std::cout << "[ANSFDBase] Liveness model loaded successfully." << std::endl;
return true;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSFDBase::LoadLivenessModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFDBase::InitializeLivenessModel(std::string licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
_faceAttrModelFolder.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSFDBase::InitializeLivenessModel", "Invalid License", __FILE__, __LINE__);
return false;
}
//2. Check if model zip file exists
if (!FileExist(modelZipFilePath)) {
this->_logger.LogFatal("ANSFDBase::InitializeLivenessModel", "Face Liveness model zip file does not exist", __FILE__, __LINE__);
return false;
}
//3. Unzip model file
//_engineType = ANSLicenseHelper::CheckHardwareInformation();
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
std::vector<std::string> passwordArray = { modelZipPassword,
"Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==",
"3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==",
"AnsCustomModels20@$",
"AnsDemoModels20@!" };
bool extractionSuccess = false;
for (const auto& password : passwordArray) {
if (ExtractPasswordProtectedZip(modelZipFilePath, password, modelName, _faceAttrModelFolder, false)) {
extractionSuccess = true;
break; // Break on successful extraction
}
}
if (!extractionSuccess || !std::filesystem::exists(_faceAttrModelFolder)) {
this->_logger.LogError("ANSFDBase::InitializeLivenessModel. Face Attribute model folder does not exist", _modelFolder, __FILE__, __LINE__);
return false;
}
_useTvDetector = false;
#ifdef USE_TV_MODEL
// We can check to load TV model here (optional)
ANSCENTER::ModelConfig _tvModelConfig;
std::string tvLabelMap;
std::string tvModelPath = CreateFilePath(_faceAttrModelFolder, "screen.onnx");
if (FileExist(tvModelPath)) {
engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation();// EngineType::CPU;//
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
// Use TensorRT YoloV11
_tvDetector = std::make_unique <ANSCENTER::ANSYOLOV10RTOD>();
}
else {
// Use OpenVINO YoloV11
_tvDetector = std::make_unique <ANSCENTER::ANSOYOLOV10OVOD>();
}
_tvModelConfig.detectionScoreThreshold = 0.6;
_tvModelConfig.modelConfThreshold = 0.5;
_tvModelConfig.modelMNSThreshold = 0.5;
if (!_tvDetector->LoadModelFromFolder("", _tvModelConfig, "screen", "screen.names", _faceAttrModelFolder, tvLabelMap)) {
this->_logger.LogError("ANSFDBase::InitializeLivenessModel", "Failed to load TV model from folder", __FILE__, __LINE__);
_useTvDetector= false;
}
else _useTvDetector = true;
}
// Perform tv model optimization if needed
std::string optimizedModelFolder;
if (_useTvDetector && (engineType == ANSCENTER::EngineType::NVIDIA_GPU)) {
this->_tvDetector->OptimizeModel(true, optimizedModelFolder);
// Zip the optimized model folder
std::string livenessModelFile = "C:\\ProgramData\\ANSCENTER\\Shared\\ANSFaceAttributesModel_Optimized.zip";
if (!FileExist(livenessModelFile)) {
if (FolderExist(optimizedModelFolder)) {
std::string modelZipPassword = "Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==";
ZipFolderWithPassword(optimizedModelFolder.c_str(), livenessModelFile.c_str(), modelZipPassword.c_str());
}
}
}
#endif
std::string antiSpoofModelPath = CreateFilePath(_faceAttrModelFolder, "liveness.onnx");
if (!FileExist(antiSpoofModelPath)) {
this->_logger.LogFatal("ANSFDBase::InitializeLivenessModel", "Liveness ONNX model file does not exist", __FILE__, __LINE__);
return false;
}
return LoadLivenessModel(antiSpoofModelPath, true);
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSFDBase::InitializeLivenessModel", e.what(), __FILE__, __LINE__);
Cleanup(); // Ensure cleanup on failure
return false;
}
}
std::pair<int, float> ANSFDBase::PredictLiveness(const cv::Mat& image) {
std::lock_guard<std::recursive_mutex> guard(_mutex);
try {
// 1. Resize
cv::Mat faceFrame = image.clone();
cv::resize(faceFrame, faceFrame, cv::Size(80, 80));
// 2. Convert to float (keep BGR order and [0,255] range)
faceFrame.convertTo(faceFrame, CV_32F, 1.0);
// 3. Convert HWC to CHW
std::vector<float> input_tensor_values(3 * 80 * 80);
int idx = 0;
for (int c = 0; c < 3; c++) {
for (int y = 0; y < 80; y++) {
for (int x = 0; x < 80; x++) {
input_tensor_values[idx++] = faceFrame.at<cv::Vec3f>(y, x)[c];
}
}
}
std::vector<int64_t> input_shape = { 1, 3, 80, 80 };
Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
Ort::Value input_tensor = Ort::Value::CreateTensor<float>(
memory_info, input_tensor_values.data(), input_tensor_values.size(),
input_shape.data(), input_shape.size()
);
std::vector<const char*> input_names_arr{ _livenessInputName.c_str() };
std::vector<const char*> output_names_arr{ _livenessOutputName.c_str() };
auto output_tensors = _livenessSession->Run(
Ort::RunOptions{ nullptr },
input_names_arr.data(), &input_tensor, 1,
output_names_arr.data(), 1
);
auto livenessResult = LivenessPostProcessing(output_tensors.front().GetTensorMutableData<float>());
float bliveness = livenessResult.first;
float conf = livenessResult.second;
faceFrame.release();
return std::pair<int, float>{bliveness, conf};
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::PredictLiveness", e.what(), __FILE__, __LINE__);
return std::pair<int, float>{ -1, 0.0f };
}
}
std::pair<int, float> ANSFDBase::LivenessPostProcessing(const float* pOutput) {
try {
float max_val = *std::max_element(pOutput, pOutput + 3), sum = 0.0f, s[3];
for (int i = 0; i < 3; ++i) sum += (s[i] = std::exp(pOutput[i] - max_val));
for (int i = 0; i < 3; ++i) s[i] /= sum;
auto max_it = std::max_element(s, s + 3);
int label = static_cast<int>(std::distance(s, max_it));
float confidence = *max_it;
if (label != 1 && confidence > 0.6f) label = 0;
return { label, confidence };
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::LivenessPostProcessing", e.what(), __FILE__, __LINE__);
return std::pair<int, float>{ -1, 0.0f };
}
}
template<typename MapTrackData, typename MapMissingFrames>
void ANSFDBase::CleanUpTracks(std::vector<Object>& currentObjects,
MapTrackData& trackDataMap,
MapMissingFrames& missingFramesMap,
const int maxMissing,
const int maxTracks)
{
std::set<int> currentIds;
for (const auto& obj : currentObjects) {
currentIds.insert(obj.trackId);
missingFramesMap[obj.trackId] = 0;
}
// Increase missing counters
for (auto& [trackId, count] : missingFramesMap) {
if (currentIds.find(trackId) == currentIds.end()) {
count++;
}
}
// Remove stale tracks
for (auto it = missingFramesMap.begin(); it != missingFramesMap.end();) {
if (it->second >= maxMissing) {
trackDataMap.erase(it->first);
it = missingFramesMap.erase(it);
}
else {
++it;
}
}
// Enforce max track limit
while (trackDataMap.size() > static_cast<size_t>(maxTracks)) {
auto it = trackDataMap.begin(); // could remove lowest ID or use heuristic
missingFramesMap.erase(it->first);
trackDataMap.erase(it);
}
}
float ANSFDBase::ComputeIoU(const cv::Rect& a, const cv::Rect& b) {
int x1 = std::max(a.x, b.x);
int y1 = std::max(a.y, b.y);
int x2 = std::min(a.x + a.width, b.x + b.width);
int y2 = std::min(a.y + a.height, b.y + b.height);
int nInterWidth = std::max(0, x2 - x1);
int nInterHeight = std::max(0, y2 - y1);
int nInterArea = nInterWidth * nInterHeight;
int nUnionArea = a.width * a.height + b.width * b.height - nInterArea;
if (nUnionArea <= 0) return 0.0f;
return static_cast<float>(nInterArea) / static_cast<float>(nUnionArea);
}
std::vector<Object> ANSFDBase::TrackFaces(const cv::Mat& inputImage,const std::vector<Object> &faceObjects) {
if (faceObjects.empty()) return faceObjects;
if (!_facelivenessEngineValid) return faceObjects;
std::vector<Object> vDetectedFaces;
vDetectedFaces.reserve(faceObjects.size());
std::lock_guard<std::recursive_mutex> guard(_mutex);
try {
std::vector<ANSCENTER::TrackerObject> vTrackerFace;
//1. Prepare detected faces for tracking
for (auto& obj : faceObjects) {
vDetectedFaces.push_back(obj);
// Update the tracker with the detected faces
float pad_track_ratio = PAD_TRACK_RATIO;
float pad_track_x = obj.box.width * pad_track_ratio;
float pad_track_y = obj.box.height * pad_track_ratio;
const int& class_id = obj.classId;
const float& prob = obj.confidence;
float x = std::max(0.0f, static_cast<float>(obj.box.x) - pad_track_x);
float y = std::max(0.0f, static_cast<float>(obj.box.y) - pad_track_y);
float width = std::min(obj.box.width + 2 * pad_track_x,
static_cast<float>(inputImage.cols) - std::max(0.0f, obj.box.x - pad_track_x));
float height = std::min(obj.box.height + 2 * pad_track_y,
static_cast<float>(inputImage.rows) - std::max(0.0f, obj.box.y - pad_track_y));
const float& left = x;
const float& top = y;
const float& right = x + width;
const float& bottom = y + height;
//// Create a ByteTrack::Object from the ANSCENTER::Object
TrackerObject temp;
temp.track_id = 0; // Placeholder, will be assigned by the tracker
temp.prob = prob;
temp.class_id = class_id;
temp.x = left;
temp.y = top;
temp.width = width;
temp.height = height;
temp.left = left;
temp.top = top;
temp.right = right;
temp.bottom = bottom;
temp.object_id = obj.cameraId;
vTrackerFace.push_back(temp);
}
//2. Update tracker and get tracked objects
std::vector<TrackerObject> vObjTracker;
UpdateANSMOTTracker(&_faceTracker, 0, vTrackerFace, vObjTracker);
//3. Match tracker by IoU with detected face
for (const auto& trkr : vObjTracker) {
int nTrackId = trkr.track_id;
cv::Rect rTrkRect(
static_cast<int>(trkr.x),
static_cast<int>(trkr.y),
static_cast<int>(trkr.width),
static_cast<int>(trkr.height)
);
int nBestIdx = -1;
float nBestIou = 0.0f;
for (int i = 0; i < static_cast<int>(vDetectedFaces.size()); ++i) {
float pad_track_ratio = PAD_TRACK_RATIO;
int pad_x = static_cast<int>(vDetectedFaces[i].box.width * pad_track_ratio);
int pad_y = static_cast<int>(vDetectedFaces[i].box.height * pad_track_ratio);
cv::Rect paddedDetectBox(
std::max(0, vDetectedFaces[i].box.x - pad_x),
std::max(0, vDetectedFaces[i].box.y - pad_y),
std::min(vDetectedFaces[i].box.width + 2 * pad_x, inputImage.cols - std::max(0, vDetectedFaces[i].box.x - pad_x)),
std::min(vDetectedFaces[i].box.height + 2 * pad_y, inputImage.rows - std::max(0, vDetectedFaces[i].box.y - pad_y))
);
float fIou = ComputeIoU(rTrkRect, paddedDetectBox);
if (fIou > nBestIou) {
nBestIou = fIou;
nBestIdx = i;
}
}
// Only assign trackId if IoU is above threshold
if (nBestIdx != -1 && nBestIou > 0.5f) {
vDetectedFaces[nBestIdx].trackId = nTrackId;
}
}
return vDetectedFaces;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::TrackFaces", e.what(), __FILE__, __LINE__);
return vDetectedFaces;
}
}
#ifdef USE_TV_MODEL
std::vector<Object> ANSFDBase::TrackTVScreens(const std::vector<Object>& tvObjects) {
if (tvObjects.empty()) return tvObjects;
if (!_facelivenessEngineValid) return tvObjects;
std::vector<Object> vRecallScreen;
std::lock_guard<std::recursive_mutex> guard(_mutex);
try {
std::vector<Object> vDetectedScreens;
vDetectedScreens.reserve(tvObjects.size());
std::vector<ANSCENTER::TrackerObject> vTVTracker;
//1. Prepare detected faces for tracking
for (auto& obj : tvObjects) {
vDetectedScreens.push_back(obj);
// Update the tracker with the detected faces
const int& class_id = obj.classId;
const float& prob = obj.confidence;
float x = obj.box.x;
float y = obj.box.y;
float width = obj.box.width;
float height = obj.box.height;
const float& left = x;
const float& top = y;
const float& right = x + width;
const float& bottom = y + height;
//// Create a ByteTrack::Object from the ANSCENTER::Object
TrackerObject temp;
temp.track_id = 0; // Placeholder, will be assigned by the tracker
temp.prob = prob;
temp.class_id = class_id;
temp.x = left;
temp.y = top;
temp.width = width;
temp.height = height;
temp.left = left;
temp.top = top;
temp.right = right;
temp.bottom = bottom;
temp.object_id = obj.cameraId;
vTVTracker.push_back(temp);
}
//2. Update tracker and get tracked objects
std::vector<TrackerObject> vObjTracker;
UpdateANSMOTTracker(&_tvTracker, 0, vTVTracker, vObjTracker);
//3. Match tracker by IoU with detected face
for (const auto& trkr : vObjTracker) {
int nTrackId = trkr.track_id;
cv::Rect rTrkRect(
static_cast<int>(trkr.x),
static_cast<int>(trkr.y),
static_cast<int>(trkr.width),
static_cast<int>(trkr.height)
);
int nBestIdx = -1;
float nBestIou = 0.0f;
for (int i = 0; i < static_cast<int>(vDetectedScreens.size()); ++i) {
cv::Rect DetectBox(
vDetectedScreens[i].box.x,
vDetectedScreens[i].box.y,
vDetectedScreens[i].box.width,
vDetectedScreens[i].box.height);
float fIou = ComputeIoU(rTrkRect, DetectBox);
if (fIou > nBestIou) {
nBestIou = fIou;
nBestIdx = i;
}
}
// Only assign trackId if IoU is above threshold
if (nBestIdx != -1 && nBestIou > 0.5f) {
vDetectedScreens[nBestIdx].trackId = nTrackId;
}
}
for (auto& screen : vDetectedScreens) {
if (screen.trackId != -1) {
if (screen.trackId < 0) continue;
auto& ScreenHistory = _mTrackScreen[screen.trackId];
ScreenHistory.push_back(screen);
}
}
vRecallScreen.reserve(_mTrackScreen.size());
for (auto& [id, objects] : _mTrackScreen) {
if (!objects.empty()) {
vRecallScreen.push_back(objects.back());
}
}
CleanUpTracks(vDetectedScreens, _mTrackScreen, _mMissingTrackScreen, MAX_MISSING_SCREEN, 1);
return vRecallScreen;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::TrackTVScreens", e.what(), __FILE__, __LINE__);
return vRecallScreen;
}
}
bool ANSFDBase::InsideScreen(const cv::Rect& a, const cv::Rect& b) {
if (a.x <= b.x && a.y <= b.y && a.width >= b.width && a.height >= b.height)
return true;
return false;
}
#endif
std::vector<Object> ANSFDBase::ValidateLivenessFaces(const cv::Mat& inputImage,const std::vector<Object> &faceObjects, const std::string& camera_id) {
if (!_facelivenessEngineValid) return faceObjects;
if (faceObjects.empty()) return faceObjects;
std::lock_guard<std::recursive_mutex> guard(_mutex);
try {
std::vector<Object> results;
// 1. Track faces
std::vector<Object> vDetectedFaces = TrackFaces(inputImage, faceObjects);
// 2. Run model liveness detection (vDetectedFaces with trackId assigned)
for (auto& result : vDetectedFaces) {
float fpad_ratio = PAD_DETECT_RATIO;
int npad_x = static_cast<int>(result.box.width * fpad_ratio);
int npad_y = static_cast<int>(result.box.height * fpad_ratio);
cv::Rect paddedROI(
std::max(0, result.box.x - npad_x),
std::max(0, result.box.y - npad_y),
std::min(result.box.width + 2 * npad_x, inputImage.cols - std::max(0, result.box.x - npad_x)),
std::min(result.box.height + 2 * npad_y, inputImage.rows - std::max(0, result.box.y - npad_y))
);
// Crop the face with padding
cv::Rect faceROI = paddedROI & cv::Rect(0, 0, inputImage.cols, inputImage.rows);
if (faceROI.width <= 0 || faceROI.height <= 0) continue;
cv::Mat face = inputImage(faceROI).clone();
std::pair<int, float> prLiveness = PredictLiveness(face);
//Return 1 if face is real, else return 0. Use for tracking attribute liveness
int bliveness = prLiveness.first;
//Skip the empty face input
if (bliveness == -1) {
continue;
}
//Verify faces on screens(TV, laptop, etc.)
bool bisFake = false;
#ifdef USE_TV_MODEL
if (_useTvDetector) {
std::vector<Object> tvObjects= _tvDetector->RunInference(inputImage,camera_id); // We need to run inference here to get TV objects
std::vector<Object> vRecallScreen = TrackTVScreens(tvObjects);
for (const auto& obj : vRecallScreen) {
if (InsideScreen(obj.box, result.box))
bisFake = true;
}
}
#endif
// If the face (with trackId) has bliveness for MAX_HISTORY_FACE frames, we can determine real/fake
if (result.trackId != -1) {
auto& dqHistory = _mTrackHistory[result.trackId];
dqHistory.push_back(prLiveness.first);
if (dqHistory.size() >= MAX_HISTORY_FACE) {
int sumDeque = std::accumulate(dqHistory.begin(), dqHistory.end(), 0);
// Decide "Real" if enough real attributes and face not on Screen else "Fake"
if (sumDeque == MAX_HISTORY_FACE && !bisFake)
result.extraInfo = "real";
else
result.extraInfo = "fake";
results.push_back(result);
dqHistory.pop_front();
}
else {
result.extraInfo = "unknown";
results.push_back(result);
}
}
}
CleanUpTracks(vDetectedFaces, _mTrackHistory, _mMissingTrackFrames, MAX_MISSING_FACE, MAX_TRACKS);
return results;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFDBase::ValidateLivenessFaces", e.what(), __FILE__, __LINE__);
return faceObjects;
}
}
}