diff --git a/engines/ONNXEngine/ONNXEngine.cpp b/engines/ONNXEngine/ONNXEngine.cpp index f87eede..7c95c60 100644 --- a/engines/ONNXEngine/ONNXEngine.cpp +++ b/engines/ONNXEngine/ONNXEngine.cpp @@ -1044,6 +1044,60 @@ namespace ANSCENTER { } } + // ==================================================================== + // AdaFace + // ==================================================================== + + Ort::Value AdaFace::transform(const cv::Mat& mat) + { + if (mat.empty()) + throw std::runtime_error("AdaFace::transform — input is empty."); + + // AdaFace consumes BGR — OpenCV Mats are already BGR, no channel swap. + cv::Mat canvas; + cv::resize(mat, canvas, cv::Size(input_node_dims.at(3), input_node_dims.at(2))); + if (canvas.type() != CV_32FC3) + canvas.convertTo(canvas, CV_32FC3); + + utils::transform::normalize_inplace(canvas, mean_val, scale_val); + + std::vector shape = input_node_dims; + if (shape[0] == -1) shape[0] = 1; + + return utils::transform::create_tensor( + canvas, shape, *memory_info_handler, + input_values_handler, utils::transform::CHW); + } + + Ort::Value AdaFace::transformBatch(const std::vector& images) + { + if (images.empty()) + throw std::runtime_error("AdaFace::transformBatch — batch is empty."); + + const int width = input_node_dims.at(3); + const int height = input_node_dims.at(2); + std::vector batch; + batch.reserve(images.size()); + + for (const auto& mat : images) { + if (mat.empty()) + throw std::runtime_error("AdaFace::transformBatch — empty image in batch."); + cv::Mat canvas; + cv::resize(mat, canvas, cv::Size(width, height)); + if (canvas.type() != CV_32FC3) + canvas.convertTo(canvas, CV_32FC3); + utils::transform::normalize_inplace(canvas, mean_val, scale_val); + batch.push_back(std::move(canvas)); + } + + std::vector shape = input_node_dims; + shape[0] = static_cast(images.size()); + + return utils::transform::create_tensor_batch( + batch, shape, *memory_info_handler, + input_values_handler, utils::transform::CHW); + } + // ==================================================================== // GlintCosFace // ==================================================================== diff --git a/engines/ONNXEngine/ONNXEngine.h b/engines/ONNXEngine/ONNXEngine.h index 250abe0..ac1238d 100644 --- a/engines/ONNXEngine/ONNXEngine.h +++ b/engines/ONNXEngine/ONNXEngine.h @@ -674,6 +674,40 @@ namespace ANSCENTER { Ort::Value transformBatch(const std::vector& images) override; }; + // ==================================================================== + // AdaFace — face recognition + // + // Same session/inference path as GlintArcFace (detect/detectBatch are + // inherited and dispatch through the virtual transform), but AdaFace + // was trained on BGR input — the official repo flips RGB→BGR before + // normalize — so the BGR2RGB conversion is omitted: OpenCV Mats are + // already BGR. Normalization is identical: (x - 127.5) / 127.5. + // ==================================================================== + class AdaFace : public GlintArcFace + { + public: + explicit AdaFace(const std::string& _onnx_path, + unsigned int _num_threads = 1) + : GlintArcFace(_onnx_path, _num_threads) + { + } + + explicit AdaFace(const std::string& _onnx_path, + EngineType engineType, + unsigned int _num_threads = 1) + : GlintArcFace(_onnx_path, engineType, _num_threads) { + } + + ~AdaFace() override = default; + + private: + static constexpr float mean_val = 127.5f; + static constexpr float scale_val = 1.f / 127.5f; + + Ort::Value transform(const cv::Mat& mat) override; + Ort::Value transformBatch(const std::vector& images) override; + }; + // ==================================================================== // MOVINET — action recognition // ==================================================================== diff --git a/modules/ANSFR/ANSFR.cpp b/modules/ANSFR/ANSFR.cpp index 1a9e5ce..0ab83e7 100644 --- a/modules/ANSFR/ANSFR.cpp +++ b/modules/ANSFR/ANSFR.cpp @@ -4,6 +4,7 @@ #include "ANSOVFaceDetector.h" #include "SCRFDFaceDetector.h" #include "ANSFaceRecognizer.h" +#include "ONNXFaceAttributes.h" #include #include #include @@ -463,6 +464,16 @@ namespace ANSCENTER { attributeModelsSuccess = false; } + // Embedding/model consistency: stored embeddings must come from + // the recognizer model that just loaded. A v1.4 upgrade flips the + // default from ArcFace to AdaFace, which silently invalidates a + // pre-existing database — regenerate before the index builds. + if (!SyncEmbeddingsWithActiveModel()) { + LogThreadSafe("ANSFacialRecognition::LoadEngine", + "Warning: embedding/model sync failed — recognition may return " + "Unknown until faces are re-enrolled"); + } + // Reload database/embeddings if (!Reload()) { LogThreadSafe("ANSFacialRecognition::LoadEngine", @@ -818,9 +829,12 @@ namespace ANSCENTER { } bool ANSFacialRecognition::InitializeRecognizer() { try { - ANS_DBG("ANSFR", "InitializeRecognizer: starting, modelPath=%s", _recognizerFilePath.c_str()); + ANS_DBG("ANSFR", "InitializeRecognizer: starting, modelPath=%s model=%s", + _recognizerFilePath.c_str(), FaceRecognizerModelName(_requestedRecModel)); // Create recognizer instance - _recognizer = std::make_unique(); + auto recognizer = std::make_unique(); + recognizer->SetPreferredModel(_requestedRecModel); + _recognizer = std::move(recognizer); _recognizer->SetMaxSlotsPerGpu(m_maxSlotsPerGpu); // Configure model @@ -861,7 +875,13 @@ namespace ANSCENTER { return false; } - ANS_DBG("ANSFR", "InitializeRecognizer: SUCCESS"); + // Record what actually loaded — ADAFACE silently falls back to + // ARCFACE when the model pack has no adafacerecognizer.onnx. + if (auto* concrete = dynamic_cast(_recognizer.get())) { + _activeRecModel = concrete->GetActiveModel(); + } + ANS_DBG("ANSFR", "InitializeRecognizer: SUCCESS, active model=%s", + FaceRecognizerModelName(_activeRecModel)); return true; } catch (const std::exception& e) { @@ -924,8 +944,50 @@ namespace ANSCENTER { return false; } } + namespace { + // EP options for the ONNX attribute models. On NVIDIA the models are + // built into TensorRT engines through the TRT EP and cached on disk + // next to the model pack (first load pays the build, every later load + // is instant). AMD/Intel/CPU resolve to DirectML/OpenVINO/CPU inside + // BasicOrtHandler and ignore every TRT field. + OrtHandlerOptions BuildAttributeOrtOptions(EngineType engineType, + const std::string& modelFolder) { + OrtHandlerOptions opts; + if (engineType == EngineType::NVIDIA_GPU) { + opts.preferTensorRT = true; + opts.trtFP16 = true; + opts.trtEngineCacheDir = CreateFilePath(modelFolder, "TRTCache"); + std::error_code ec; + std::filesystem::create_directories(opts.trtEngineCacheDir, ec); + } + return opts; + } + } + bool ANSFacialRecognition::InitializeAgeGenderModel(const std::string& deviceName) { try { + // Prefer the ONNX model shipped since v1.4; fall back to the + // legacy OpenVINO IR when it is absent (or fails to load). + std::string onnxModelPath = CreateFilePath(_recognizerModelFolder, "agegender.onnx"); + if (FileExist(onnxModelPath)) { + try { + auto detector = std::make_unique( + onnxModelPath, engineType, + BuildAttributeOrtOptions(engineType, _recognizerModelFolder)); + + std::lock_guard lock(_ageGenderMutex); + _ageGenderDetector = std::move(detector); + LogThreadSafe("ANSFacialRecognition::InitializeAgeGenderModel", + "Loaded ONNX age-gender model: " + onnxModelPath, LogLevel::Info); + return true; + } + catch (const std::exception& e) { + LogThreadSafe("ANSFacialRecognition::InitializeAgeGenderModel", + "ONNX age-gender model failed (" + std::string(e.what()) + + ") — falling back to OpenVINO model"); + } + } + std::string ageGenderModelPath = CreateFilePath(_recognizerModelFolder, "agegender.xml"); if (!FileExist(ageGenderModelPath)) { @@ -949,18 +1011,20 @@ namespace ANSCENTER { _ageGenderDetector.reset(); // Create new detector - _ageGenderDetector = std::make_unique(ageGenderModelPath, true); + auto detector = std::make_unique(ageGenderModelPath, true); // Compile model for target device ov::CompiledModel compiledModel = core->compile_model( - _ageGenderDetector->read(*core), + detector->read(*core), deviceName ); // Create inference request - _ageGenderDetector->request = compiledModel.create_infer_request(); - _ageGenderDetector->inTensor = _ageGenderDetector->request.get_input_tensor(); - _ageGenderDetector->inTensor.set_shape(_ageGenderDetector->inShape); + detector->request = compiledModel.create_infer_request(); + detector->inTensor = detector->request.get_input_tensor(); + detector->inTensor.set_shape(detector->inShape); + + _ageGenderDetector = std::move(detector); } return true; @@ -979,6 +1043,30 @@ namespace ANSCENTER { } bool ANSFacialRecognition::InitializeEmotionModel(const std::string& deviceName) { try { + // Prefer the YOLOv11 classifier shipped since v1.4 (class list in + // emotionsrecognition.names); fall back to the OpenVINO IR. + std::string onnxModelPath = CreateFilePath(_recognizerModelFolder, "emotionsrecognition.onnx"); + if (FileExist(onnxModelPath)) { + try { + std::string namesPath = CreateFilePath(_recognizerModelFolder, "emotionsrecognition.names"); + auto classNames = OnnxEmotionsClassifier::LoadClassNames(namesPath); + auto detector = std::make_unique( + onnxModelPath, std::move(classNames), engineType, + BuildAttributeOrtOptions(engineType, _recognizerModelFolder)); + + std::lock_guard lock(_emotionsMutex); + _emotionsDetector = std::move(detector); + LogThreadSafe("ANSFacialRecognition::InitializeEmotionModel", + "Loaded ONNX emotion model: " + onnxModelPath, LogLevel::Info); + return true; + } + catch (const std::exception& e) { + LogThreadSafe("ANSFacialRecognition::InitializeEmotionModel", + "ONNX emotion model failed (" + std::string(e.what()) + + ") — falling back to OpenVINO model"); + } + } + std::string emotionModelPath = CreateFilePath(_recognizerModelFolder, "emotionsrecognition.xml"); if (!FileExist(emotionModelPath)) { @@ -1002,18 +1090,20 @@ namespace ANSCENTER { _emotionsDetector.reset(); // Create new detector - _emotionsDetector = std::make_unique(emotionModelPath, true); + auto detector = std::make_unique(emotionModelPath, true); // Compile model for target device ov::CompiledModel compiledModel = core->compile_model( - _emotionsDetector->read(*core), + detector->read(*core), deviceName ); // Create inference request - _emotionsDetector->request = compiledModel.create_infer_request(); - _emotionsDetector->inTensor = _emotionsDetector->request.get_input_tensor(); - _emotionsDetector->inTensor.set_shape(_emotionsDetector->inShape); + detector->request = compiledModel.create_infer_request(); + detector->inTensor = detector->request.get_input_tensor(); + detector->inTensor.set_shape(detector->inShape); + + _emotionsDetector = std::move(detector); } return true; @@ -1084,7 +1174,184 @@ namespace ANSCENTER { return false; } } - + + bool ANSFacialRecognition::SyncEmbeddingsWithActiveModel() { + try { + if (!_db || !_recognizer) return false; + + const std::string activeName = FaceRecognizerModelName(_activeRecModel); + std::string storedName; + std::vector faces; + { + std::lock_guard lock(_databaseMutex); + storedName = _db->GetSetting("EMBEDDING_MODEL"); + if (_db->GetAllFaces(faces) != 0) return false; + } + + if (storedName.empty()) { + // Databases created before v1.4 never recorded the model — + // every embedding in them came from ArcFace. + storedName = faces.empty() ? activeName : "arcface"; + } + + if (storedName != activeName && !faces.empty()) { + LogThreadSafe("ANSFacialRecognition::SyncEmbeddingsWithActiveModel", + "Stored embeddings were generated by '" + storedName + + "' but the active recognizer is '" + activeName + + "' — regenerating " + std::to_string(faces.size()) + + " embeddings from the enrolled face images", LogLevel::Info); + if (!RegenerateEmbeddings()) return false; + } + + { + std::lock_guard lock(_databaseMutex); + _db->SetSetting("EMBEDDING_MODEL", activeName); + } + return true; + } + catch (const std::exception& e) { + LogThreadSafe("ANSFacialRecognition::SyncEmbeddingsWithActiveModel", e.what()); + return false; + } + } + + bool ANSFacialRecognition::RegenerateEmbeddings() { + try { + if (!_db || !_recognizer || !_recognizer->IsInitialized()) { + LogThreadSafe("ANSFacialRecognition::RegenerateEmbeddings", + "Recognizer/database not ready"); + return false; + } + + std::vector faces; + { + std::lock_guard lock(_databaseMutex); + if (_db->GetAllFaces(faces) != 0) return false; + } + if (faces.empty()) return true; + + size_t updated = 0, failed = 0; + for (const auto& face : faces) { + // InsertFace stored the aligned face crop — compute the new + // embedding directly on it, no re-detection needed. + cv::Mat crop = cv::imread(face.ImagePath, cv::IMREAD_COLOR); + if (crop.empty()) { + LogThreadSafe("ANSFacialRecognition::RegenerateEmbeddings", + "Cannot read enrolled face image (face ID " + + std::to_string(face.FaceId) + "): " + face.ImagePath); + ++failed; + continue; + } + + Object faceObject; + faceObject.mask = crop; + + std::vector embedding; + { + std::lock_guard lock(_recognitionMutex); + embedding = _recognizer->Feature(crop, faceObject); + } + if (embedding.size() != static_cast(_config._recOutputDim)) { + LogThreadSafe("ANSFacialRecognition::RegenerateEmbeddings", + "Embedding failed for face ID " + std::to_string(face.FaceId)); + ++failed; + continue; + } + + std::lock_guard lock(_databaseMutex); + if (_db->UpdateFaceEmbedding(face.FaceId, embedding.data()) > 0) ++updated; + else ++failed; + } + + LogThreadSafe("ANSFacialRecognition::RegenerateEmbeddings", + "Regenerated " + std::to_string(updated) + "/" + + std::to_string(faces.size()) + " embeddings (" + + std::to_string(failed) + " failed)", LogLevel::Info); + + _reloadNeeded.store(true); + return updated > 0 || failed == 0; + } + catch (const std::exception& e) { + LogThreadSafe("ANSFacialRecognition::RegenerateEmbeddings", e.what()); + return false; + } + } + + bool ANSFacialRecognition::SetRecognizerModelType(int modelType) { + try { + if (modelType < 0 || modelType > 1) { + LogThreadSafe("ANSFacialRecognition::SetRecognizerModelType", + "Invalid model type: " + std::to_string(modelType) + + " (0 = AdaFace, 1 = ArcFace)"); + return false; + } + const auto requested = static_cast(modelType); + _requestedRecModel = requested; + + // Engine not loaded yet — the preference is applied by LoadEngine. + if (!_recognizer) return true; + + if (requested == _activeRecModel) return true; + + LogThreadSafe("ANSFacialRecognition::SetRecognizerModelType", + "Switching recognizer model to '" + + std::string(FaceRecognizerModelName(requested)) + "'", LogLevel::Info); + + if (!InitializeRecognizer()) { + LogThreadSafe("ANSFacialRecognition::SetRecognizerModelType", + "Failed to load recognizer for model '" + + std::string(FaceRecognizerModelName(requested)) + "'"); + return false; + } + _recognizerModelFolder = _recognizer->GetModelFolder(); + + if (!SyncEmbeddingsWithActiveModel()) { + LogThreadSafe("ANSFacialRecognition::SetRecognizerModelType", + "Warning: embedding regeneration failed after model switch"); + } + + _reloadNeeded.store(true); + return Reload(); + } + catch (const std::exception& e) { + LogThreadSafe("ANSFacialRecognition::SetRecognizerModelType", e.what()); + return false; + } + } + + int ANSFacialRecognition::GetRecognizerModelType() const { + return static_cast(_recognizer ? _activeRecModel : _requestedRecModel); + } + + std::string ANSFacialRecognition::GetActiveModelInfo() { + try { + const char* agegenderBackend = "disabled"; + const char* emotionBackend = "disabled"; + { + std::lock_guard lock(_ageGenderMutex); + if (_ageGenderDetector && _ageGenderDetector->enabled()) + agegenderBackend = _ageGenderDetector->backendName(); + } + { + std::lock_guard lock(_emotionsMutex); + if (_emotionsDetector && _emotionsDetector->enabled()) + emotionBackend = _emotionsDetector->backendName(); + } + + nlohmann::json info; + info["recognizer_model"] = FaceRecognizerModelName( + _recognizer ? _activeRecModel : _requestedRecModel); + info["recognizer_loaded"] = (_recognizer != nullptr); + info["agegender_backend"] = agegenderBackend; + info["emotion_backend"] = emotionBackend; + return info.dump(); + } + catch (const std::exception& e) { + LogThreadSafe("ANSFacialRecognition::GetActiveModelInfo", e.what()); + return "{}"; + } + } + bool ANSFacialRecognition::UpdateParameters( float knownPersonThreshold, bool enableAgeGender, diff --git a/modules/ANSFR/ANSFR.h b/modules/ANSFR/ANSFR.h index 31aa6ad..85388cf 100644 --- a/modules/ANSFR/ANSFR.h +++ b/modules/ANSFR/ANSFR.h @@ -97,6 +97,17 @@ namespace ANSCENTER bool UpdateFaceQueue(int queueSize, int faceThresholdSize, bool enableFaceQueue); bool GetFaceQueue(int& queueSize, int& faceThresholdSize, bool& enableFaceQueue); + + // Recognizer model switching (0 = AdaFace [default], 1 = ArcFace). + // Before LoadEngine: just records the preference. After LoadEngine: + // re-initializes the recognizer, regenerates every stored embedding + // from the enrolled face crops (embeddings from different models are + // not comparable), and rebuilds the FAISS index. + bool SetRecognizerModelType(int modelType); + int GetRecognizerModelType() const; + // JSON summary of what is actually loaded: recognizer model plus the + // backend ("onnx" | "openvino" | "disabled") serving each attribute. + [[nodiscard]] std::string GetActiveModelInfo(); [[nodiscard]] std::vector Recognize(const cv::Mat& frame); // Expect to return on 1 object [[nodiscard]] std::vector Inference(const cv::Mat& input); [[nodiscard]] std::vector Detect(const cv::Mat& input); @@ -141,6 +152,11 @@ namespace ANSCENTER bool InitializeAgeGenderModel(const std::string& deviceName); bool InitializeEmotionModel(const std::string& deviceName); bool InitializeHeadPoseModel(const std::string& deviceName); + // Regenerates stored embeddings when they were produced by a + // different recognizer model than the one currently loaded + // (tracked via the EMBEDDING_MODEL key in the face database). + bool SyncEmbeddingsWithActiveModel(); + bool RegenerateEmbeddings(); void ensureUniqueUserIdWithHighestConfidence(std::vector& resultObjects, CameraData* camData = nullptr); Object GetLargestObject(const std::vector& objects); bool AreFacesSimilar(const FaceResultObject& face1, const FaceResultObject& face2); @@ -214,9 +230,13 @@ namespace ANSCENTER std::unique_ptr_recognizer = nullptr; std::unique_ptr_detector = nullptr; std::unique_ptr_antiSpoofDetector = nullptr; - std::unique_ptr_ageGenderDetector = nullptr; + std::unique_ptr_ageGenderDetector = nullptr; std::unique_ptr_headPoseDetector = nullptr; - std::unique_ptr_emotionsDetector = nullptr; + std::unique_ptr_emotionsDetector = nullptr; + // Recognizer model preference (settable before/after LoadEngine) + // and what actually loaded (fallback may downgrade ADAFACE->ARCFACE). + FaceRecognizerModel _requestedRecModel{ FaceRecognizerModel::ADAFACE }; + FaceRecognizerModel _activeRecModel{ FaceRecognizerModel::ADAFACE }; const size_t QUEUE_SIZE = 10; const size_t FACE_THRESHOLD_SIZE = 5; // Cached attribute JSON per tracked face (everything except head pose). @@ -348,6 +368,23 @@ extern "C" ANSFR_API int GetParamters(ANSCENTER::ANSFacialRecognition** Handl extern "C" ANSFR_API int UpdateFaceQueue(ANSCENTER::ANSFacialRecognition** Handle, int queueSize, int numKnownFaceInQueue, int enableFaceQueue); extern "C" ANSFR_API int GetFaceQueue(ANSCENTER::ANSFacialRecognition** Handle, LStrHandle faceQueue); +// Recognizer model switching (ANS_FaceRecognizer v1.4+). +// modelType: 0 = AdaFace (default), 1 = ArcFace. +// Call before LoadANSRFEngine to pick the model for the initial load, or +// after it to switch live — a live switch reloads the recognizer engine and +// regenerates every stored face embedding from the enrolled face images +// (embeddings produced by different models are not comparable), so it can +// take a while on large databases. Returns 1 on success, 0 on failure, -1 on +// invalid arguments. +extern "C" ANSFR_API int SetANSRFRecognizerModel(ANSCENTER::ANSFacialRecognition** Handle, int modelType); +// Returns the ACTIVE model (0 = AdaFace, 1 = ArcFace) or -1 on error. May +// differ from what was requested when the model pack has no +// adafacerecognizer.onnx (pre-v1.4 zips fall back to ArcFace). +extern "C" ANSFR_API int GetANSRFRecognizerModel(ANSCENTER::ANSFacialRecognition** Handle); +// JSON describing the loaded models, e.g. +// {"recognizer_model":"adaface","agegender_backend":"onnx","emotion_backend":"onnx",...} +extern "C" ANSFR_API int GetANSRFModelInfo(ANSCENTER::ANSFacialRecognition** Handle, LStrHandle modelInfo); + extern "C" ANSFR_API int GetUserString(ANSCENTER::ANSFacialRecognition * *Handle, int userId, std::string& userRecord); diff --git a/modules/ANSFR/ANSFRCommon.cpp b/modules/ANSFR/ANSFRCommon.cpp index fcf97b4..ba2cca3 100644 --- a/modules/ANSFR/ANSFRCommon.cpp +++ b/modules/ANSFR/ANSFRCommon.cpp @@ -654,10 +654,14 @@ namespace ANSCENTER { adjustedEmotionName = "Attentive"; else if (emotionName == "Surprised" || emotionName == "surprised") adjustedEmotionName = "Attentive"; + else if (emotionName == "Surprise" || emotionName == "surprise") + adjustedEmotionName = "Attentive"; else if (emotionName == "Disgusted" || emotionName == "disgusted") adjustedEmotionName = "Attentive"; else if (emotionName == "Fearful" || emotionName == "fearful") adjustedEmotionName = "Attentive"; + else if (emotionName == "Fear" || emotionName == "fear") + adjustedEmotionName = "Attentive"; else if (emotionName == "Anger" || emotionName == "anger") adjustedEmotionName = "Attentive"; else diff --git a/modules/ANSFR/ANSFRCommon.h b/modules/ANSFR/ANSFRCommon.h index 07bb3cc..158d483 100644 --- a/modules/ANSFR/ANSFRCommon.h +++ b/modules/ANSFR/ANSFRCommon.h @@ -107,6 +107,42 @@ namespace ANSCENTER { float age; float maleProb; }; + + // Which face-embedding model the recognizer runs. ADAFACE is the default + // since ANS_FaceRecognizer v1.4 (adafacerecognizer.onnx); when that file + // is missing from the model zip the recognizer silently falls back to + // ARCFACE (ansfacerecognizer.onnx) so v1.3-and-older zips keep working. + enum class FaceRecognizerModel : int { + ADAFACE = 0, + ARCFACE = 1, + }; + + inline const char* FaceRecognizerModelName(FaceRecognizerModel m) { + return (m == FaceRecognizerModel::ARCFACE) ? "arcface" : "adaface"; + } + + // ---------------------------------------------------------------- + // Attribute-model backends. The v1.4 model pack ships ONNX attribute + // models (agegender.onnx / emotionsrecognition.onnx) next to the + // legacy OpenVINO IR pairs (.xml/.bin). These interfaces let the + // pipeline hold either backend behind the same two calls it already + // makes: runInfer() and enabled(). + // ---------------------------------------------------------------- + class ANSF_API IAgeGenderEstimator { + public: + virtual ~IAgeGenderEstimator() = default; + virtual AgeGenderResults runInfer(const cv::Mat& frame) = 0; + virtual bool enabled() const = 0; + virtual const char* backendName() const = 0; // "openvino" | "onnx" + }; + + class ANSF_API IEmotionsClassifier { + public: + virtual ~IEmotionsClassifier() = default; + virtual std::map runInfer(const cv::Mat& frame) = 0; + virtual bool enabled() const = 0; + virtual const char* backendName() const = 0; // "openvino" | "onnx" + }; //// Class for face attribute class ANSF_API Face { public: @@ -200,7 +236,7 @@ namespace ANSCENTER { //std::vector getAllResults(); }; - class ANSF_API AgeGenderDetection :public BaseDetection { + class ANSF_API AgeGenderDetection : public BaseDetection, public IAgeGenderEstimator { public: std::recursive_mutex _mutex; std::string outputAge; @@ -211,7 +247,9 @@ namespace ANSCENTER { bool doRawOutputMessages); virtual std::shared_ptr read(const ov::Core& core) override; - AgeGenderResults runInfer(const cv::Mat& frame); + AgeGenderResults runInfer(const cv::Mat& frame) override; + bool enabled() const override { return BaseDetection::enabled(); } + const char* backendName() const override { return "openvino"; } //void submitRequest(); //void enqueue(const cv::Mat& face); //AgeGenderResults getResult(int idx); @@ -219,7 +257,7 @@ namespace ANSCENTER { }; - class ANSF_API EmotionsDetection : public BaseDetection { + class ANSF_API EmotionsDetection : public BaseDetection, public IEmotionsClassifier { public: std::recursive_mutex _mutex; //size_t enquedFaces; @@ -230,7 +268,9 @@ namespace ANSCENTER { bool doRawOutputMessages); virtual std::shared_ptr read(const ov::Core& core) override; - std::map runInfer(const cv::Mat& frame); + std::map runInfer(const cv::Mat& frame) override; + bool enabled() const override { return BaseDetection::enabled(); } + const char* backendName() const override { return "openvino"; } //void submitRequest(); //void enqueue(const cv::Mat& face); //std::map getResult(int idx); diff --git a/modules/ANSFR/ANSFaceRecognizer.cpp b/modules/ANSFR/ANSFaceRecognizer.cpp index 781f243..900efc7 100644 --- a/modules/ANSFR/ANSFaceRecognizer.cpp +++ b/modules/ANSFR/ANSFaceRecognizer.cpp @@ -87,6 +87,31 @@ namespace { } namespace ANSCENTER { + std::string ANSFaceRecognizer::ResolveRecognizerModelFile() { + const std::string adaPath = CreateFilePath(_modelFolder, "adafacerecognizer.onnx"); + const std::string arcPath = CreateFilePath(_modelFolder, "ansfacerecognizer.onnx"); + + if (_preferredModel == FaceRecognizerModel::ADAFACE) { + if (std::filesystem::exists(adaPath)) { + _activeModel = FaceRecognizerModel::ADAFACE; + m_swapRB = false; // AdaFace consumes BGR + ANS_DBG("FaceRecognizer", "Recognizer model resolved: ADAFACE (%s)", adaPath.c_str()); + return adaPath; + } + _logger.LogWarn("ANSFaceRecognizer::ResolveRecognizerModelFile", + "adafacerecognizer.onnx not found in model pack — falling back to ArcFace", + __FILE__, __LINE__); + } + + _activeModel = FaceRecognizerModel::ARCFACE; + m_swapRB = true; // ArcFace consumes RGB + if (std::filesystem::exists(arcPath)) { + ANS_DBG("FaceRecognizer", "Recognizer model resolved: ARCFACE (%s)", arcPath.c_str()); + return arcPath; + } + return std::string(); + } + std::string ANSFaceRecognizer::GetOpenVINODevice() { ov::Core core; std::vector available_devices = core.get_available_devices(); @@ -146,16 +171,16 @@ namespace ANSCENTER { static_cast(EngineType::AMD_GPU), static_cast(EngineType::CPU)); if (engineType == EngineType::NVIDIA_GPU) { - // 1. Load ONNX model - std::string onnxfile = CreateFilePath(_modelFolder, "ansfacerecognizer.onnx"); - if (std::filesystem::exists(onnxfile)) { + // 1. Load ONNX model (adaface by default, arcface fallback) + std::string onnxfile = ResolveRecognizerModelFile(); + if (!onnxfile.empty()) { _modelFilePath = onnxfile; - _logger.LogDebug("ANSFaceRecognizer::Initialize. Loading ansfacerecognizer weight", + _logger.LogDebug("ANSFaceRecognizer::Initialize. Loading face recognizer weight", _modelFilePath, __FILE__, __LINE__); } else { - _logger.LogError("ANSFaceRecognizer::Initialize. Model ansfacerecognizer.onnx does not exist", - _modelFilePath, __FILE__, __LINE__); + _logger.LogError("ANSFaceRecognizer::Initialize. No recognizer model (adafacerecognizer.onnx / ansfacerecognizer.onnx) exists", + _modelFolder, __FILE__, __LINE__); return false; } @@ -213,24 +238,28 @@ namespace ANSCENTER { } else { #ifdef USE_ONNX_ENGINE - std::string faceidModel = CreateFilePath(_modelFolder, "ansfacerecognizer.onnx"); - if (std::filesystem::exists(faceidModel)) { + std::string faceidModel = ResolveRecognizerModelFile(); + if (!faceidModel.empty()) { _modelFilePath = faceidModel; _logger.LogDebug("ANSFaceRecognizer::Initialize. Loading ANSFaceRecognizer weight", _modelFilePath, __FILE__, __LINE__); } else { - _logger.LogError("ANSFaceRecognizer::Initialize. Model ansfacerecognizer.onnx does not exist", - _modelFilePath, __FILE__, __LINE__); + _logger.LogError("ANSFaceRecognizer::Initialize. No recognizer model (adafacerecognizer.onnx / ansfacerecognizer.onnx) exists", + _modelFolder, __FILE__, __LINE__); return false; } - ANS_DBG("FaceRecognizer", "Creating GlintArcFace with engineType=%d model=%s", + ANS_DBG("FaceRecognizer", "Creating %s with engineType=%d model=%s", + FaceRecognizerModelName(_activeModel), static_cast(engineType), faceidModel.c_str()); - faceRecognizer = std::make_unique(faceidModel, engineType); + if (_activeModel == FaceRecognizerModel::ADAFACE) + faceRecognizer = std::make_unique(faceidModel, engineType); + else + faceRecognizer = std::make_unique(faceidModel, engineType); if (!faceRecognizer) { _logger.LogFatal("ANSFaceRecognizer::Initialize", "Failed to initialize ONNX face recognizer", __FILE__, __LINE__); - ANS_DBG("FaceRecognizer", "FAILED: GlintArcFace returned null"); + ANS_DBG("FaceRecognizer", "FAILED: ONNX face recognizer returned null"); return false; } #else @@ -279,29 +308,29 @@ namespace ANSCENTER { if (!result) return false; if (engineType == EngineType::NVIDIA_GPU) { - std::string onnxfile = CreateFilePath(_modelFolder, "ansfacerecognizer.onnx"); - if (std::filesystem::exists(onnxfile)) { + std::string onnxfile = ResolveRecognizerModelFile(); + if (!onnxfile.empty()) { _modelFilePath = onnxfile; - _logger.LogDebug("ANSFaceRecognizer::LoadModel. Loading ansfacerecognizer weight", + _logger.LogDebug("ANSFaceRecognizer::LoadModel. Loading face recognizer weight", _modelFilePath, __FILE__, __LINE__); } else { - _logger.LogError("ANSFaceRecognizer::LoadModel. ansfacerecognizer.onnx not found", - _modelFilePath, __FILE__, __LINE__); + _logger.LogError("ANSFaceRecognizer::LoadModel. No recognizer model (adafacerecognizer.onnx / ansfacerecognizer.onnx) found", + _modelFolder, __FILE__, __LINE__); return false; } } else { - std::string faceidModel = CreateFilePath(_modelFolder, "ansfacerecognizer.onnx"); + std::string faceidModel = ResolveRecognizerModelFile(); #ifdef USE_ONNX_ENGINE - if (std::filesystem::exists(faceidModel)) { + if (!faceidModel.empty()) { _modelFilePath = faceidModel; - _logger.LogDebug("ANSFaceRecognizer::LoadModel. Loading ansfacerecognizer weight", + _logger.LogDebug("ANSFaceRecognizer::LoadModel. Loading face recognizer weight", _modelFilePath, __FILE__, __LINE__); } else { - _logger.LogError("ANSFaceRecognizer::LoadModel. ansfacerecognizer.onnx not found", - _modelFilePath, __FILE__, __LINE__); + _logger.LogError("ANSFaceRecognizer::LoadModel. No recognizer model (adafacerecognizer.onnx / ansfacerecognizer.onnx) found", + _modelFolder, __FILE__, __LINE__); return false; } #else @@ -834,7 +863,7 @@ namespace ANSCENTER { } try { - // CPU preprocessing: resize + BGR→RGB before GPU upload + // CPU preprocessing: resize + channel order before GPU upload // Reduces PCIe transfer and eliminates GPU cvtColor/resize overhead cv::Mat srcImg = inputImage; if (srcImg.channels() == 1) { @@ -846,8 +875,13 @@ namespace ANSCENTER { } else { cpuResized = srcImg; } + // ArcFace needs RGB; AdaFace consumes the Mat's native BGR. cv::Mat cpuRGB; - cv::cvtColor(cpuResized, cpuRGB, cv::COLOR_BGR2RGB); + if (m_swapRB) { + cv::cvtColor(cpuResized, cpuRGB, cv::COLOR_BGR2RGB); + } else { + cpuRGB = cpuResized; + } m_gpuRgb.upload(cpuRGB, *m_gpuStream); m_gpuStream->waitForCompletion(); @@ -963,14 +997,19 @@ namespace ANSCENTER { cv::cuda::resize(d_img, d_resized, targetSize, 0, 0, cv::INTER_LINEAR, *m_gpuStream); d_img = d_resized; } - cv::cuda::GpuMat d_rgb; - cv::cuda::cvtColor(d_img, d_rgb, cv::COLOR_BGR2RGB, 0, *m_gpuStream); - batchGpu.emplace_back(std::move(d_rgb)); + // ArcFace needs RGB; AdaFace consumes native BGR. + if (m_swapRB) { + cv::cuda::GpuMat d_rgb; + cv::cuda::cvtColor(d_img, d_rgb, cv::COLOR_BGR2RGB, 0, *m_gpuStream); + batchGpu.emplace_back(std::move(d_rgb)); + } else { + batchGpu.emplace_back(std::move(d_img)); + } } else { const auto& roi = faceROIs[i]; if (roi.empty()) continue; - // CPU preprocessing: resize + BGR→RGB before upload + // CPU preprocessing: resize + channel order before upload cv::Mat srcImg = roi; if (srcImg.channels() == 1) { cv::cvtColor(srcImg, srcImg, cv::COLOR_GRAY2BGR); @@ -982,7 +1021,11 @@ namespace ANSCENTER { cpuResized = srcImg; } cv::Mat cpuRGB; - cv::cvtColor(cpuResized, cpuRGB, cv::COLOR_BGR2RGB); + if (m_swapRB) { + cv::cvtColor(cpuResized, cpuRGB, cv::COLOR_BGR2RGB); + } else { + cpuRGB = cpuResized; + } cv::cuda::GpuMat d_rgb; d_rgb.upload(cpuRGB, *m_gpuStream); batchGpu.emplace_back(std::move(d_rgb)); diff --git a/modules/ANSFR/ANSFaceRecognizer.h b/modules/ANSFR/ANSFaceRecognizer.h index 32992f4..9745eba 100644 --- a/modules/ANSFR/ANSFaceRecognizer.h +++ b/modules/ANSFR/ANSFaceRecognizer.h @@ -53,6 +53,12 @@ namespace ANSCENTER { int GetEmbeddingSize() const { return FACE_EMBEDDING_SIZE; } std::shared_ptr GetGpuResources() const { return m_gpuResources; } + // Which embedding model to load (call BEFORE Initialize). Default is + // ADAFACE; Initialize falls back to ARCFACE when the model pack has + // no adafacerecognizer.onnx (pre-v1.4 zips). + void SetPreferredModel(FaceRecognizerModel m) { _preferredModel = m; } + FaceRecognizerModel GetActiveModel() const { return _activeModel; } + bool UpdateParamater(double knownPersonThreshold) { _modelConfig.unknownPersonThreshold = knownPersonThreshold; m_knownPersonThresh = _modelConfig.unknownPersonThreshold; @@ -68,6 +74,11 @@ namespace ANSCENTER { private: bool LoadEngine(const std::string& xmlModelPath, bool engineOptimisation = true); + // Resolves the .onnx file for _preferredModel, falling back + // ADAFACE -> ARCFACE when adafacerecognizer.onnx is absent. Sets + // _activeModel and m_swapRB; returns "" when nothing usable exists. + std::string ResolveRecognizerModelFile(); + // Batched forward: one embedding per Object.mask (caller must hold _mutex) std::vector> ForwardUnlocked(const cv::Mat& input,const std::vector& outputBbox); @@ -127,6 +138,13 @@ namespace ANSCENTER { float m_knownPersonThresh = 0.35f; EngineType engineType; + FaceRecognizerModel _preferredModel = FaceRecognizerModel::ADAFACE; + FaceRecognizerModel _activeModel = FaceRecognizerModel::ADAFACE; + // ArcFace was trained on RGB (needs the BGR->RGB swap); AdaFace on + // BGR (OpenCV native order, no swap). Gates every cvtColor in the + // TensorRT preprocessing paths. + bool m_swapRB = true; + #ifdef USE_ONNX_ENGINE std::unique_ptr faceRecognizer = nullptr; const int CPU_FACE_WIDTH = 112; diff --git a/modules/ANSFR/CMakeLists.txt b/modules/ANSFR/CMakeLists.txt index 5920839..4f04406 100644 --- a/modules/ANSFR/CMakeLists.txt +++ b/modules/ANSFR/CMakeLists.txt @@ -4,6 +4,7 @@ set(ANSFR_SOURCES ANSFR.cpp ANSFRCommon.cpp ANSFaceRecognizer.cpp + ONNXFaceAttributes.cpp ANSGpuFrameRegistry.cpp GpuNV12SlotPool.cpp FaceDatabase.cpp diff --git a/modules/ANSFR/FaceDatabase.cpp b/modules/ANSFR/FaceDatabase.cpp index d4c6608..1f37b80 100644 --- a/modules/ANSFR/FaceDatabase.cpp +++ b/modules/ANSFR/FaceDatabase.cpp @@ -265,6 +265,85 @@ namespace ANSCENTER { } } + int FaceDatabase::GetAllFaces(std::vector& faces) { + std::lock_guard lock(_mutex); + faces.clear(); + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(m_db, "SELECT ID, USER, IMG_PATH FROM FACE;", -1, &stmt, NULL); + if (rc != SQLITE_OK) { + _logger.LogError("FaceDatabase::GetAllFaces", sqlite3_errmsg(m_db), __FILE__, __LINE__); + return -1; + } + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + FaceRecord record; + record.FaceId = sqlite3_column_int(stmt, 0); + record.UserId = sqlite3_column_int(stmt, 1); + const unsigned char* path = sqlite3_column_text(stmt, 2); + record.ImagePath = path ? reinterpret_cast(path) : ""; + faces.push_back(std::move(record)); + } + sqlite3_finalize(stmt); + return 0; + } + int FaceDatabase::UpdateFaceEmbedding(int faceId, const float embedding[]) { + std::lock_guard lock(_mutex); + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(m_db, "UPDATE FACE SET EMBEDDING=? WHERE ID=?;", -1, &stmt, NULL); + if (rc != SQLITE_OK) { + _logger.LogError("FaceDatabase::UpdateFaceEmbedding", sqlite3_errmsg(m_db), __FILE__, __LINE__); + return -1; + } + int resultCode = 1; + if (sqlite3_bind_blob(stmt, 1, embedding, sizeof(float) * m_embedding_dim, SQLITE_STATIC) != SQLITE_OK) + resultCode = -2; + if (sqlite3_bind_int(stmt, 2, faceId) != SQLITE_OK) + resultCode = -2; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW && rc != SQLITE_DONE) { + _logger.LogError("FaceDatabase::UpdateFaceEmbedding", sqlite3_errmsg(m_db), __FILE__, __LINE__); + resultCode = -3; + } + sqlite3_finalize(stmt); + return resultCode; + } + bool FaceDatabase::SetSetting(const std::string& key, const std::string& value) { + std::lock_guard lock(_mutex); + // Older databases predate the SETTINGS table — create on demand. + sqlite3_exec(m_db, + "CREATE TABLE IF NOT EXISTS SETTINGS (KEY TEXT PRIMARY KEY, VALUE TEXT);", + nullptr, nullptr, nullptr); + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(m_db, + "INSERT INTO SETTINGS (KEY, VALUE) VALUES (?, ?) " + "ON CONFLICT(KEY) DO UPDATE SET VALUE=excluded.VALUE;", + -1, &stmt, NULL); + if (rc != SQLITE_OK) { + _logger.LogError("FaceDatabase::SetSetting", sqlite3_errmsg(m_db), __FILE__, __LINE__); + return false; + } + sqlite3_bind_text(stmt, 1, key.c_str(), int(key.length()), SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, value.c_str(), int(value.length()), SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + return (rc == SQLITE_ROW || rc == SQLITE_DONE); + } + std::string FaceDatabase::GetSetting(const std::string& key) { + std::lock_guard lock(_mutex); + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(m_db, "SELECT VALUE FROM SETTINGS WHERE KEY=?;", -1, &stmt, NULL); + if (rc != SQLITE_OK) { + // Missing table (legacy database) — treat as "no value". + return std::string(); + } + std::string value; + sqlite3_bind_text(stmt, 1, key.c_str(), int(key.length()), SQLITE_STATIC); + if (sqlite3_step(stmt) == SQLITE_ROW) { + const unsigned char* v = sqlite3_column_text(stmt, 0); + if (v) value = reinterpret_cast(v); + } + sqlite3_finalize(stmt); + return value; + } int FaceDatabase::DeleteFace(int id) { try { // Fix #4: Single lock scope — GetFaceById + DELETE in one atomic section diff --git a/modules/ANSFR/FaceDatabase.h b/modules/ANSFR/FaceDatabase.h index bf3528c..2959cb0 100644 --- a/modules/ANSFR/FaceDatabase.h +++ b/modules/ANSFR/FaceDatabase.h @@ -35,6 +35,15 @@ namespace ANSCENTER { int GetUsersWithFaceIds(std::vector& ids, std::vector& codes, std::vector& userNames, std::vector>& faceIdsByUser); int GetFaceById(int faceId, int& userId, std::string& imagePath); + // All enrolled faces (ID, USER, IMG_PATH) — used to regenerate + // embeddings after a recognizer model switch. + int GetAllFaces(std::vector& faces); + // Replaces the stored embedding of one face (same dim as Initialize). + int UpdateFaceEmbedding(int faceId, const float embedding[]); + // Simple key/value store (SETTINGS table, created on demand) — used + // to persist which recognizer model generated the stored embeddings. + bool SetSetting(const std::string& key, const std::string& value); + std::string GetSetting(const std::string& key); std::map GetUserDict(); int GetEmbeddings(ANSFRBase& recognizer); int GetEmbeddings(std::unique_ptr& recognizer); diff --git a/modules/ANSFR/ONNXFaceAttributes.cpp b/modules/ANSFR/ONNXFaceAttributes.cpp new file mode 100644 index 0000000..cdf6867 --- /dev/null +++ b/modules/ANSFR/ONNXFaceAttributes.cpp @@ -0,0 +1,256 @@ +#include "ONNXFaceAttributes.h" +#include +#include +#include +#include + +namespace { + + // The exported classifier may or may not end with a softmax node + // (Ultralytics kept flipping this across versions). Treat the output + // as probabilities only when it already looks like a distribution. + void SoftmaxIfNeeded(std::vector& v) { + if (v.empty()) return; + float sum = 0.0f; + bool inRange = true; + for (float x : v) { + if (x < 0.0f || x > 1.0f) { inRange = false; break; } + sum += x; + } + if (inRange && std::fabs(sum - 1.0f) < 0.05f) return; + + float maxVal = *std::max_element(v.begin(), v.end()); + float denom = 0.0f; + for (float& x : v) { + x = std::exp(x - maxVal); + denom += x; + } + if (denom > 0.0f) { + for (float& x : v) x /= denom; + } + } + + void ResolveInputSize(const std::vector& dims, int& width, int& height) { + // NCHW; dynamic dims come back as -1 — the shipped models are 224x224. + width = 224; + height = 224; + if (dims.size() >= 4) { + if (dims[2] > 0) height = static_cast(dims[2]); + if (dims[3] > 0) width = static_cast(dims[3]); + } + } + +} + +namespace ANSCENTER { + + // ================================================================ + // OnnxAgeGenderEstimator + // ================================================================ + + OnnxAgeGenderEstimator::OnnxAgeGenderEstimator(const std::string& onnxPath, + EngineType engineType, + const OrtHandlerOptions& options, + unsigned int numThreads) + : BasicOrtHandler(onnxPath, engineType, options, numThreads) + { + _ready = true; + } + + Ort::Value OnnxAgeGenderEstimator::transform(const cv::Mat& mat) + { + if (mat.empty()) + throw std::runtime_error("OnnxAgeGenderEstimator::transform — input is empty."); + + int width = 0, height = 0; + ResolveInputSize(input_node_dims, width, height); + + cv::Mat canvas = mat; + if (canvas.channels() == 1) + cv::cvtColor(canvas, canvas, cv::COLOR_GRAY2BGR); + cv::Mat resized; + cv::resize(canvas, resized, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::cvtColor(resized, resized, cv::COLOR_BGR2RGB); + resized.convertTo(resized, CV_32FC3, 1.0 / 255.0); + + // ImageNet normalization (ViTFeatureExtractor defaults) + static const float mean_vals[3] = { 0.485f, 0.456f, 0.406f }; + static const float scale_vals[3] = { 1.f / 0.229f, 1.f / 0.224f, 1.f / 0.225f }; + utils::transform::normalize_inplace(resized, mean_vals, scale_vals); + + std::vector shape = input_node_dims; + if (shape.size() < 4) shape = { 1, 3, height, width }; + if (shape[0] <= 0) shape[0] = 1; + if (shape[2] <= 0) shape[2] = height; + if (shape[3] <= 0) shape[3] = width; + + return utils::transform::create_tensor( + resized, shape, *memory_info_handler, + input_values_handler, utils::transform::CHW); + } + + Ort::Value OnnxAgeGenderEstimator::transformBatch(const std::vector&) + { + throw std::runtime_error("OnnxAgeGenderEstimator::transformBatch — not supported."); + } + + AgeGenderResults OnnxAgeGenderEstimator::runInfer(const cv::Mat& frame) + { + std::lock_guard lock(_mutex); + try { + if (!_ready || frame.empty()) return { -1, -1 }; + + Ort::Value input_tensor = transform(frame); + DmlInferenceGuard dmlGuard(m_engineType, "OnnxAgeGenderEstimator::runInfer"); + auto output_tensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), &input_tensor, 1, + output_node_names.data(), num_outputs); + + // Two known export layouts, distinguished by tensor sizes: + // (a) v1.4 pack: age_pred [1] (years) + gender_logits [1,2] + // with class order [male, female] + // (b) research export: a single [1,2] tensor of + // [age_years, gender_logit] where sigmoid >= 0.5 -> Female + const float* agePtr = nullptr; + const float* genderLogits = nullptr; + for (auto& tensor : output_tensors) { + const size_t count = + tensor.GetTensorTypeAndShapeInfo().GetElementCount(); + if (count == 1 && !agePtr) + agePtr = tensor.GetTensorMutableData(); + else if (count >= 2 && !genderLogits) + genderLogits = tensor.GetTensorMutableData(); + } + + if (agePtr && genderLogits) { // layout (a) + const float age = std::clamp(agePtr[0], 0.0f, 100.0f); + const float m = genderLogits[0], f = genderLogits[1]; + const float mx = std::max(m, f); + const float em = std::exp(m - mx), ef = std::exp(f - mx); + return { age, em / (em + ef) }; + } + + if (genderLogits) { // layout (b) + const float age = std::clamp(genderLogits[0], 0.0f, 100.0f); + const float femaleProb = 1.0f / (1.0f + std::exp(-genderLogits[1])); + return { age, 1.0f - femaleProb }; + } + + return { -1, -1 }; + } + catch (const std::exception& error) { + std::cerr << "OnnxAgeGenderEstimator error: " << error.what() << std::endl; + return { -1, -1 }; + } + } + + // ================================================================ + // OnnxEmotionsClassifier + // ================================================================ + + std::vector OnnxEmotionsClassifier::LoadClassNames(const std::string& namesFilePath) + { + std::vector names; + try { + std::ifstream file(namesFilePath); + std::string line; + while (std::getline(file, line)) { + while (!line.empty() && (line.back() == '\r' || line.back() == '\n' + || line.back() == ' ' || line.back() == '\t')) + line.pop_back(); + if (!line.empty()) names.push_back(line); + } + } + catch (const std::exception&) { + names.clear(); + } + if (names.empty()) { + names = { "neutral", "happy", "sad", "surprise", "anger" }; + } + return names; + } + + OnnxEmotionsClassifier::OnnxEmotionsClassifier(const std::string& onnxPath, + std::vector classNames, + EngineType engineType, + const OrtHandlerOptions& options, + unsigned int numThreads) + : BasicOrtHandler(onnxPath, engineType, options, numThreads) + , _classNames(std::move(classNames)) + { + if (_classNames.empty()) + _classNames = { "neutral", "happy", "sad", "surprise", "anger" }; + _ready = true; + } + + Ort::Value OnnxEmotionsClassifier::transform(const cv::Mat& mat) + { + if (mat.empty()) + throw std::runtime_error("OnnxEmotionsClassifier::transform — input is empty."); + + int width = 0, height = 0; + ResolveInputSize(input_node_dims, width, height); + + cv::Mat canvas = mat; + if (canvas.channels() == 1) + cv::cvtColor(canvas, canvas, cv::COLOR_GRAY2BGR); + cv::Mat resized; + cv::resize(canvas, resized, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::cvtColor(resized, resized, cv::COLOR_BGR2RGB); + // Ultralytics classifier preprocessing: RGB, scaled to [0, 1] + resized.convertTo(resized, CV_32FC3, 1.0 / 255.0); + + std::vector shape = input_node_dims; + if (shape.size() < 4) shape = { 1, 3, height, width }; + if (shape[0] <= 0) shape[0] = 1; + if (shape[2] <= 0) shape[2] = height; + if (shape[3] <= 0) shape[3] = width; + + return utils::transform::create_tensor( + resized, shape, *memory_info_handler, + input_values_handler, utils::transform::CHW); + } + + Ort::Value OnnxEmotionsClassifier::transformBatch(const std::vector&) + { + throw std::runtime_error("OnnxEmotionsClassifier::transformBatch — not supported."); + } + + std::map OnnxEmotionsClassifier::runInfer(const cv::Mat& frame) + { + std::lock_guard lock(_mutex); + try { + if (!_ready || frame.empty()) return { }; + + Ort::Value input_tensor = transform(frame); + DmlInferenceGuard dmlGuard(m_engineType, "OnnxEmotionsClassifier::runInfer"); + auto output_tensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), &input_tensor, 1, + output_node_names.data(), num_outputs); + + auto info = output_tensors.at(0).GetTensorTypeAndShapeInfo(); + const size_t count = info.GetElementCount(); + if (count == 0) return { }; + + const float* vals = output_tensors.at(0).GetTensorMutableData(); + std::vector scores(vals, vals + count); + SoftmaxIfNeeded(scores); + + std::map emotions; + for (size_t i = 0; i < scores.size(); ++i) { + const std::string name = (i < _classNames.size()) + ? _classNames[i] + : ("class_" + std::to_string(i)); + emotions[name] = scores[i]; + } + return emotions; + } + catch (const std::exception& error) { + std::cerr << "OnnxEmotionsClassifier error: " << error.what() << std::endl; + return { }; + } + } + +} diff --git a/modules/ANSFR/ONNXFaceAttributes.h b/modules/ANSFR/ONNXFaceAttributes.h new file mode 100644 index 0000000..f50c01d --- /dev/null +++ b/modules/ANSFR/ONNXFaceAttributes.h @@ -0,0 +1,69 @@ +#ifndef ONNXFACEATTRIBUTES_H +#define ONNXFACEATTRIBUTES_H +#pragma once +// ONNX Runtime backends for the face attribute models shipped with +// ANS_FaceRecognizer v1.4: +// - agegender.onnx ViT-Base age/gender head (224x224, ImageNet norm, +// 2 logits: [age 0..100, gender sigmoid -> Female >= 0.5]) +// - emotionsrecognition.onnx YOLOv11 classifier (class list in +// emotionsrecognition.names) +// +// Execution provider follows the detected hardware through BasicOrtHandler: +// NVIDIA -> TensorRT EP (disk-cached engines) with CUDA EP fallback, +// AMD -> DirectML, Intel -> OpenVINO, otherwise CPU. + +#include "ANSFRCommon.h" +#include "ONNXEngine.h" +#include + +namespace ANSCENTER { + + class OnnxAgeGenderEstimator : public BasicOrtHandler, public IAgeGenderEstimator { + public: + OnnxAgeGenderEstimator(const std::string& onnxPath, + EngineType engineType, + const OrtHandlerOptions& options, + unsigned int numThreads = 2); + ~OnnxAgeGenderEstimator() override = default; + + AgeGenderResults runInfer(const cv::Mat& frame) override; + bool enabled() const override { return _ready; } + const char* backendName() const override { return "onnx"; } + + private: + Ort::Value transform(const cv::Mat& mat) override; + Ort::Value transformBatch(const std::vector& images) override; + + std::recursive_mutex _mutex; + bool _ready{ false }; + }; + + class OnnxEmotionsClassifier : public BasicOrtHandler, public IEmotionsClassifier { + public: + OnnxEmotionsClassifier(const std::string& onnxPath, + std::vector classNames, + EngineType engineType, + const OrtHandlerOptions& options, + unsigned int numThreads = 2); + ~OnnxEmotionsClassifier() override = default; + + std::map runInfer(const cv::Mat& frame) override; + bool enabled() const override { return _ready; } + const char* backendName() const override { return "onnx"; } + + // Reads one class name per line; returns the legacy 5-emotion list + // when the file is missing/empty so the classifier still produces + // labelled output. + static std::vector LoadClassNames(const std::string& namesFilePath); + + private: + Ort::Value transform(const cv::Mat& mat) override; + Ort::Value transformBatch(const std::vector& images) override; + + std::recursive_mutex _mutex; + std::vector _classNames; + bool _ready{ false }; + }; + +} +#endif diff --git a/modules/ANSFR/dllmain.cpp b/modules/ANSFR/dllmain.cpp index 8eef0cc..36f7396 100644 --- a/modules/ANSFR/dllmain.cpp +++ b/modules/ANSFR/dllmain.cpp @@ -1294,6 +1294,42 @@ extern "C" ANSFR_API int GetFaceQueue(ANSCENTER::ANSFacialRecognition** Handl } } +extern "C" ANSFR_API int SetANSRFRecognizerModel(ANSCENTER::ANSFacialRecognition** Handle, int modelType) { + ANS_DBG("ANSFR","SetANSRFRecognizerModel: HandlePtr=%p, *Handle=%p, modelType=%d (0=AdaFace,1=ArcFace)", + (void*)Handle, (void*)(Handle ? *Handle : nullptr), modelType); + try { + if (!Handle || !*Handle) return -1; + if (modelType < 0 || modelType > 1) return -1; + FRHandleGuard guard(AcquireFRHandle(*Handle)); + if (!guard) return -1; + return (*Handle)->SetRecognizerModelType(modelType) ? 1 : 0; + } + catch (const std::exception& e) { + ANS_DBG("ANSFR","SetANSRFRecognizerModel EXCEPTION: %s", e.what()); + return -1; + } +} + +extern "C" ANSFR_API int GetANSRFRecognizerModel(ANSCENTER::ANSFacialRecognition** Handle) { + try { + if (!Handle || !*Handle) return -1; + return (*Handle)->GetRecognizerModelType(); + } + catch (const std::exception&) { + return -1; + } +} + +extern "C" ANSFR_API int GetANSRFModelInfo(ANSCENTER::ANSFacialRecognition** Handle, LStrHandle modelInfo) { + try { + if (!Handle || !*Handle || !modelInfo) return -1; + return CopyToLStrHandle(modelInfo, (*Handle)->GetActiveModelInfo()); + } + catch (const std::exception&) { + return -1; + } +} + extern "C" ANSFR_API int RunInferenceComplete_LV( ANSCENTER::ANSFacialRecognition** Handle, cv::Mat** cvImage, diff --git a/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp b/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp index 303112b..0db29df 100644 --- a/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp +++ b/tests/ANSFR-UnitTest/ANSFR-UnitTest.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include "ANSFR.h" #include "fastdeploy/vision.h" #include "ANSFilePlayer.h" @@ -1713,6 +1714,117 @@ int SCRFDVideoRepro() return 0; } +// =========================================================================== +// ANS_FaceRecognizer v1.4 test: +// - AdaFace loads by default from the v1.4 model pack +// - ONNX age-gender + emotion models load (TensorRT EP on NVIDIA) +// - SetANSRFRecognizerModel switches adaface <-> arcface live, and stored +// embeddings are regenerated so recognition keeps working after a switch +// =========================================================================== +static bool V14Check(bool condition, const std::string& label, int& failures) +{ + std::cout << (condition ? "[PASS] " : "[FAIL] ") << label << std::endl; + if (!condition) ++failures; + return condition; +} + +int TestModelSwitchV14() +{ + const std::string modelZip = "C:\\Programs\\Models\\ANS_FaceRecognizer_v1.4.zip"; + const std::string detectorZip = "C:\\ProgramData\\ANSCENTER\\ANSVIS Server\\ANSFDET\\ANS_GenericFD(GPU)_v1.0.zip"; + const std::string dbFolder = "C:\\Programs\\DemoAssets\\ANSFRDB\\v14_test"; + const std::string dbFile = dbFolder + "\\ANSFR_v14_test.db"; + const std::string enrollImage = "C:\\Programs\\DemoAssets\\TestImages\\FacialRecognition\\Obama+Oprah1.jpg"; + const std::string userName = "V14TestUser"; + + int failures = 0; + std::cout << "\n===== TestModelSwitchV14 =====" << std::endl; + + // Fresh database each run so the test is repeatable + std::error_code ec; + std::filesystem::remove_all(dbFolder, ec); + + cv::Mat frame = cv::imread(enrollImage, cv::IMREAD_COLOR); + if (!V14Check(!frame.empty(), "test image readable: " + enrollImage, failures)) + return failures; + + ANSCENTER::ANSFacialRecognition* h = nullptr; + int cr = CreateANSRFHandle(&h, "", "", dbFile.c_str(), modelZip.c_str(), detectorZip.c_str(), + 0, 0.35f, 1, 1, 1, 30, 0.5f, 0, 0); + if (!V14Check(cr > 0 && h != nullptr, "CreateANSRFHandle", failures)) + return failures; + + int le = LoadANSRFEngine(&h); + if (!V14Check(le == 1, "LoadANSRFEngine (v1.4 model pack)", failures)) { + ReleaseANSRFHandle(&h); + return failures; + } + + // --- default model + attribute backends ------------------------------- + std::string info = h->GetActiveModelInfo(); + std::cout << "Model info: " << info << std::endl; + V14Check(GetANSRFRecognizerModel(&h) == 0, "default recognizer model is AdaFace (0)", failures); + V14Check(info.find("\"recognizer_model\":\"adaface\"") != std::string::npos, + "model info reports adaface", failures); + V14Check(info.find("\"agegender_backend\":\"onnx\"") != std::string::npos, + "age-gender uses agegender.onnx backend", failures); + V14Check(info.find("\"emotion_backend\":\"onnx\"") != std::string::npos, + "emotions use emotionsrecognition.onnx backend", failures); + + // --- enroll one user --------------------------------------------------- + int userId = InsertUser(&h, "0001", userName.c_str()); + V14Check(userId > 0, "InsertUser", failures); + + unsigned int bufLen = 0; + unsigned char* buf = ANSCENTER::ANSFRHelper::CVMatToBytes(frame, bufLen); + int faceInserted = InsertFaceBinary(&h, userId, buf, frame.cols, frame.rows); + delete buf; + V14Check(faceInserted > 0, "InsertFaceBinary (enrollment)", failures); + Reload(&h); + + auto recognizesUser = [&](const char* label) { + // Full pipeline (detect + match). The Recognize API needs a config + // file with _recInputShape, which this test intentionally omits. + unsigned int len = 0; + unsigned char* bytes = ANSCENTER::ANSFRHelper::CVMatToBytes(frame, len); + std::string res = RunANSRFInferenceBinary(&h, bytes, frame.cols, frame.rows); + delete bytes; + std::cout << " recognition result: " << res << std::endl; + V14Check(res.find(userName) != std::string::npos, label, failures); + }; + + recognizesUser("recognized with AdaFace embeddings"); + + // --- live switch to ArcFace (regenerates stored embeddings) ------------ + V14Check(SetANSRFRecognizerModel(&h, 1) == 1, "switch to ArcFace", failures); + V14Check(GetANSRFRecognizerModel(&h) == 1, "active model is ArcFace (1)", failures); + std::cout << "Model info: " << h->GetActiveModelInfo() << std::endl; + recognizesUser("recognized after switch to ArcFace"); + + // --- and back to AdaFace ------------------------------------------------ + V14Check(SetANSRFRecognizerModel(&h, 0) == 1, "switch back to AdaFace", failures); + V14Check(GetANSRFRecognizerModel(&h) == 0, "active model is AdaFace (0)", failures); + recognizesUser("recognized after switch back to AdaFace"); + + // --- attributes through the full pipeline (ONNX age/gender + emotions) -- + { + unsigned int len = 0; + unsigned char* bytes = ANSCENTER::ANSFRHelper::CVMatToBytes(frame, len); + std::string res = RunANSRFInferenceBinary(&h, bytes, frame.cols, frame.rows); + delete bytes; + std::cout << " inference (attributes): " << res << std::endl; + V14Check(res.find("Age") != std::string::npos || res.find("results") != std::string::npos, + "inference with ONNX attribute models returns results", failures); + } + + ReleaseANSRFHandle(&h); + + std::cout << "===== TestModelSwitchV14: " + << (failures == 0 ? "ALL PASSED" : (std::to_string(failures) + " FAILURE(S)")) + << " =====" << std::endl; + return failures; +} + int main(int argc, char** argv) { // ── GPU visibility bisect (Blackwell Myelin l2cm debugging) ───────────── @@ -1735,7 +1847,8 @@ int main(int argc, char** argv) //FaceDetectorTest(); //TestFaceRecognition(); //TestCompleteFR1(); - ANSVISTest(); + TestModelSwitchV14(); + //ANSVISTest(); //ANSVISImageTest(); //ANSVISTest(); //ANSVISTestFilePlayer();