Files
ANSCORE/modules/ANSFR/ANSFRCommon.cpp
T
nghianguyenandClaude Fable 5 430c8057b8 ANSFR: report v1.4 emotion class names verbatim in the attribute JSON
The legacy display remap in FaceAttributeToJsonString rewrote most
emotions to "Attentive", which hid the v1.4 classifier's real classes.
The remap now applies ONLY to the legacy OpenVINO model's lowercase
classes (sad/happy/neutral/anger/surprise - v1.1 output unchanged);
every other name, i.e. the classes read from emotionsrecognition.names
(Angry, Fear, Happy, Neutral, Sad), passes through verbatim. A retrained
classifier with a different .names list needs no code change.

Verified on the classroom video: faces previously labelled "Attentive"
now read "Angry (100%)"; smiling faces read "Happy (100%)". Parity test
updated to check each pack against its own emotion vocabulary.

Note for client code: v1.4 deployments no longer emit "Attentive" -
anything matching that string must handle Angry/Fear/Sad instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-28 16:50:47 +10:00

1165 lines
47 KiB
C++

#include "ANSFRCommon.h"
#include "dirent.h"
#include <ctime>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/dnn/dnn.hpp>
#include <opencv2/highgui.hpp>
#include <utils/ocv_common.hpp>
#include <json.hpp>
namespace ANSCENTER {
constexpr size_t ndetections = 200;
template <typename T> T GetData(const boost::property_tree::ptree& pt, const std::string& key)
{
T ret;
if (boost::optional<T> data = pt.get_optional<T>(key))
{
ret = data.get();
}
return ret;
}
Face::Face(size_t id, cv::Rect& location) :
_location(location), _intensity_mean(0.f), _id(id), _age(-1),
_maleScore(0), _femaleScore(0), _realFaceConfidence(0),
_isAgeGenderEnabled(false), _isEmotionsEnabled(false),
_isHeadPoseEnabled(false), _isLandmarksEnabled(false), _isAntispoofingEnabled(false)
{
HeadPoseResults headPose;
headPose.angle_p = 0.f;
headPose.angle_r = 0.f;
headPose.angle_y = 0.f;
_headPose = headPose;
}
void Face::updateAge(float value) {
_age = (_age == -1) ? value : 0.95f * _age + 0.05f * value;
}
void Face::updateGender(float value) {
if (value < 0)
return;
if (value > 0.5) {
_maleScore += value - 0.5f;
}
else {
_femaleScore += 0.5f - value;
}
}
void Face::updateEmotions(std::map<std::string, float> values) {
for (auto& kv : values) {
if (_emotions.find(kv.first) == _emotions.end()) {
_emotions[kv.first] = kv.second;
}
else {
_emotions[kv.first] = 0.9f * _emotions[kv.first] + 0.1f * kv.second;
}
}
}
void Face::updateHeadPose(HeadPoseResults values) {
_headPose = values;
}
void Face::updateLandmarks(std::vector<float> values) {
_landmarks = std::move(values);
}
void Face::updateRealFaceConfidence(float value) {
_realFaceConfidence = value;
}
void Face::updateFaceLiveness(int value) {
_faceLiveness = value;
}
int Face::getAge() {
return static_cast<int>(std::floor(_age + 0.5f));
}
bool Face::isMale() {
return _maleScore > _femaleScore;
}
bool Face::isReal() {
return _realFaceConfidence > 0.4;
}
float Face::getAntispoofingScore() {
return _realFaceConfidence;
}
int Face::getFaceLiveness() {
return _faceLiveness;
}
std::map<std::string, float> Face::getEmotions() {
return _emotions;
}
std::pair<std::string, float> Face::getMainEmotion() {
auto x = std::max_element(_emotions.begin(), _emotions.end(),
[](const std::pair<std::string, float>& p1, const std::pair<std::string, float>& p2) {
return p1.second < p2.second; });
return std::make_pair(x->first, x->second);
}
HeadPoseResults Face::getHeadPose() {
return _headPose;
}
const std::vector<float>& Face::getLandmarks() {
return _landmarks;
}
size_t Face::getId() {
return _id;
}
void Face::ageGenderEnable(bool value) {
_isAgeGenderEnabled = value;
}
void Face::emotionsEnable(bool value) {
_isEmotionsEnabled = value;
}
void Face::headPoseEnable(bool value) {
_isHeadPoseEnabled = value;
}
void Face::landmarksEnable(bool value) {
_isLandmarksEnabled = value;
}
void Face::antispoofingEnable(bool value) {
_isAntispoofingEnabled = value;
}
void Face::faceLivenessEnable(bool value) {
_isFaceLivenessEnabled = value;
}
bool Face::isAgeGenderEnabled() {
return _isAgeGenderEnabled;
}
bool Face::isEmotionsEnabled() {
return _isEmotionsEnabled;
}
bool Face::isHeadPoseEnabled() {
return _isHeadPoseEnabled;
}
bool Face::isLandmarksEnabled() {
return _isLandmarksEnabled;
}
bool Face::isAntispoofingEnabled() {
return _isAntispoofingEnabled;
}
bool Face::isFaceLivenessEnabled() {
return _isFaceLivenessEnabled;
}
float calcIoU(cv::Rect& src, cv::Rect& dst) {
cv::Rect i = src & dst;
cv::Rect u = src | dst;
return static_cast<float>(i.area()) / static_cast<float>(u.area());
}
float calcMean(const cv::Mat& src) {
cv::Mat tmp;
cv::cvtColor(src, tmp, cv::COLOR_BGR2GRAY);
cv::Scalar mean = cv::mean(tmp);
return static_cast<float>(mean[0]);
}
Face matchFace(cv::Rect rect, std::list<Face>& faces) {
Face face(0,rect);
float maxIoU = 0.55f;
for (auto&& f : faces) {
float iou = calcIoU(rect, f._location);
if (iou > maxIoU) {
face = f;
maxIoU = iou;
}
}
return face;
}
BaseDetection::BaseDetection(const std::string& pathToModel, bool doRawOutputMessages)
: pathToModel(pathToModel), doRawOutputMessages(doRawOutputMessages) {
}
bool BaseDetection::enabled() const {
return bool(request);
}
AntispoofingClassifier::AntispoofingClassifier(const std::string& pathToModel, bool doRawOutputMessages)
: BaseDetection(pathToModel, doRawOutputMessages)
{
_modelFilePath = pathToModel;
}
float AntispoofingClassifier::runInfer(const cv::Mat& frame) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
cv::Mat face = frame.clone();
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
resize2tensor(face, inSlice);
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
request.infer();
face.release();
float r = request.get_output_tensor().data<const float>()[0] * 100;
return r;
}
catch (const std::exception& error) {
std::cerr << "Error: " << error.what() << std::endl;
return -1;
}
}
std::shared_ptr<ov::Model> AntispoofingClassifier::read(const ov::Core& core) {
slog::info << "Reading model: " << _modelFilePath << slog::endl;
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
ov::preprocess::PrePostProcessor ppp(model);
ppp.input().tensor().
set_element_type(ov::element::u8).
set_layout("NHWC");
ppp.input().preprocess().convert_layout("NCHW");
ppp.output().tensor().set_element_type(ov::element::f32);
model = ppp.build();
inShape = model->input().get_shape();
inShape[0] = ndetections;
//ov::set_batch(model, { 1, int64_t(ndetections) });
ov::set_batch(model, { 1 }); // Force single batch for all inputs
return model;
}
AgeGenderDetection::AgeGenderDetection(const std::string& pathToModel,
bool doRawOutputMessages)
: BaseDetection(pathToModel, doRawOutputMessages)
{
_modelFilePath = pathToModel;
}
AgeGenderResults AgeGenderDetection::runInfer(const cv::Mat& frame) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
cv::Mat face = frame.clone();
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
resize2tensor(face, inSlice);
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
request.infer();
face.release();
AgeGenderResults r = { request.get_tensor(outputAge).data<const float>()[0] * 100,
request.get_tensor(outputGender).data<const float>()[0 * 2 + 1] };
return r;
}
catch (const std::exception& error) {
AgeGenderResults r = { -1, -1 };
return r;
}
}
std::shared_ptr<ov::Model> AgeGenderDetection::read(const ov::Core& core) {
slog::info << "Reading model: " << _modelFilePath << slog::endl;
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
outputAge = "age_conv3";
outputGender = "prob";
ov::preprocess::PrePostProcessor ppp(model);
ppp.input().tensor().
set_element_type(ov::element::u8).
set_layout("NHWC");
ppp.input().preprocess().
convert_element_type(ov::element::f32).
convert_layout("NCHW");
ppp.output(outputAge).tensor().set_element_type(ov::element::f32);
ppp.output(outputGender).tensor().set_element_type(ov::element::f32);
model = ppp.build();
inShape = model->input().get_shape();
inShape[0] = ndetections;
//ov::set_batch(model, { 1, int64_t(ndetections) });
ov::set_batch(model, { 1 }); // Force single batch for all inputs
return model;
}
HeadPoseDetection::HeadPoseDetection(const std::string& pathToModel,
bool doRawOutputMessages)
: BaseDetection(pathToModel, doRawOutputMessages),
outputAngleR("angle_r_fc"), outputAngleP("angle_p_fc"), outputAngleY("angle_y_fc")
{
_modelFilePath = pathToModel;
}
HeadPoseResults HeadPoseDetection::runInfer(const cv::Mat& frame) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
cv::Mat face = frame.clone();
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
resize2tensor(face, inSlice);
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
request.infer();
face.release();
HeadPoseResults r = { request.get_tensor(outputAngleR).data<const float>()[0],
request.get_tensor(outputAngleP).data<const float>()[0],
request.get_tensor(outputAngleY).data<const float>()[0] };
return r;
}
catch (const std::exception& error) {
HeadPoseResults r = { -1, -1, -1 };
return r;
}
}
std::shared_ptr<ov::Model> HeadPoseDetection::read(const ov::Core& core) {
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
ov::preprocess::PrePostProcessor ppp(model);
ppp.input().tensor().
set_element_type(ov::element::u8).
set_layout("NHWC");
ppp.input().preprocess().convert_layout("NCHW");
ppp.output(outputAngleR).tensor().set_element_type(ov::element::f32);
ppp.output(outputAngleP).tensor().set_element_type(ov::element::f32);
ppp.output(outputAngleY).tensor().set_element_type(ov::element::f32);
model = ppp.build();
inShape = model->input().get_shape();
inShape[0] = ndetections;
//ov::set_batch(model, { 1, int64_t(ndetections) });
ov::set_batch(model, { 1 }); // Force single batch for all inputs
return model;
}
EmotionsDetection::EmotionsDetection(const std::string& pathToModel,
bool doRawOutputMessages)
: BaseDetection(pathToModel, doRawOutputMessages)
{
_modelFilePath = pathToModel;
}
std::map<std::string, float> EmotionsDetection::runInfer(const cv::Mat& frame) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
cv::Mat face = frame.clone();
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
resize2tensor(face, inSlice);
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
request.infer();
face.release();
auto tensor = request.get_output_tensor();
auto emotionsVecSize = emotionsVec.size();
size_t numOfChannels = tensor.get_shape().at(1);
if (numOfChannels == emotionsVecSize) {
const float* emotionsValues = tensor.data<const float>();
auto outputIdxPos = emotionsValues + 0 * emotionsVecSize;
std::map<std::string, float> emotions;
for (size_t i = 0; i < emotionsVecSize; i++) {
emotions[emotionsVec[i]] = outputIdxPos[i];
}
return emotions;
}
else return { };
}
catch (const std::exception& error) {
std::cerr << "Error: " << error.what() << std::endl;
return { };
}
}
std::shared_ptr<ov::Model> EmotionsDetection::read(const ov::Core& core) {
slog::info << "Reading model: " << _modelFilePath << slog::endl;
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
ov::preprocess::PrePostProcessor ppp(model);
ppp.input().tensor().
set_element_type(ov::element::u8).
set_layout("NHWC");
ppp.input().preprocess().convert_layout("NCHW");
ppp.output().tensor().set_element_type(ov::element::f32);
model = ppp.build();
inShape = model->input().get_shape();
inShape[0] = ndetections;
//ov::set_batch(model, { 1, int64_t(ndetections) });
ov::set_batch(model, { 1 }); // Force single batch for all inputs
return model;
}
FacialLandmarksDetection::FacialLandmarksDetection(const std::string& pathToModel,
bool doRawOutputMessages)
: BaseDetection(pathToModel, doRawOutputMessages)
{
_modelFilePath = pathToModel;
}
std::vector<float> FacialLandmarksDetection::runInfer(const cv::Mat& frame) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<float> normedLandmarks;
try {
cv::Mat face = frame.clone();
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
resize2tensor(face, inSlice);
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
request.infer();
face.release();
auto tensor = request.get_output_tensor();
size_t n_lm = tensor.get_shape().at(1);
const float* normed_coordinates = tensor.data<const float>();
auto begin = 0;
auto end = begin + n_lm / 2;
for (auto i_lm = begin; i_lm < end; ++i_lm) {
float normed_x = normed_coordinates[2 * i_lm];
float normed_y = normed_coordinates[2 * i_lm + 1];
normedLandmarks.push_back(normed_x);
normedLandmarks.push_back(normed_y);
}
return normedLandmarks;
}
catch (const std::exception& error) {
std::cerr << "Error: " << error.what() << std::endl;
return normedLandmarks;
}
}
std::shared_ptr<ov::Model> FacialLandmarksDetection::read(const ov::Core& core) {
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
ov::Shape outShape = model->output().get_shape();
if (outShape.size() != 2 && outShape.back() != 70) {
return model;
}
ov::preprocess::PrePostProcessor ppp(model);
ppp.input().tensor().
set_element_type(ov::element::u8).
set_layout("NHWC");
ppp.input().preprocess().convert_layout("NCHW");
ppp.output().tensor().set_element_type(ov::element::f32);
model = ppp.build();
inShape = model->input().get_shape();
inShape[0] = ndetections;
//ov::set_batch(model, { 1, int64_t(ndetections) });
ov::set_batch(model, { 1 }); // Force single batch for all inputs
return model;
}
//void ANSFRHelper::ANSFRHelper::CheckCudaStatus(cudaError_t status) {
// try {
// if (status != cudaSuccess) {
// std::cerr << "CUDA API failed with status " << status << ": " << cudaGetErrorString(status) << std::endl;
// }
// }
// catch (std::exception& e) {
// std::cerr << "ANSFRHelper::CheckCublasStatus" << e.what();
// }
//}
//void ANSFRHelper::ANSFRHelper::CheckCublasStatus(cublasStatus_t status) {
// try {
// if (status != CUBLAS_STATUS_SUCCESS) {
// std::cerr << "cuBLAS API failed with status " << status << std::endl;
// }
// }
// catch (std::exception& e) {
// std::cerr << "ANSFRHelper::CheckCublasStatus" << e.what();
// }
//}
void ANSFRHelper::GetCroppedFaces(const cv::Mat &input, std::vector<Object>& outputBbox, int resize_w, int resize_h, std::vector<struct CroppedFace>& croppedFaces) {
croppedFaces.clear();
cv::Mat frame = input.clone();
if (outputBbox.size() > 0) {
for (std::vector<Object>::iterator it = outputBbox.begin(); it != outputBbox.end(); it++) {
int x1, y1, x2, y2;
x1 = (*it).box.x;
y1 = (*it).box.y;
x2 = (*it).box.x + (*it).box.width;
y2 = (*it).box.y + (*it).box.height;
cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2));
cv::Mat tempCrop = frame(facePos);
struct CroppedFace currFace;
cv::resize(tempCrop, currFace.faceMat, cv::Size(resize_h, resize_w), 0, 0, cv::INTER_CUBIC);
currFace.face = currFace.faceMat.clone();
currFace.x1 = x1;
currFace.y1 = y1;
currFace.x2 = x2;
currFace.y2 = y2;
croppedFaces.push_back(currFace);
}
}
else {
// this will be the same image
int x1, y1, x2, y2;
x1 = 0;
y1 = 0;
x2 = frame.cols;
y2 = frame.rows;
cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2));
cv::Mat tempCrop = frame(facePos);
struct CroppedFace currFace;
cv::resize(tempCrop, currFace.faceMat, cv::Size(resize_h, resize_w), 0, 0, cv::INTER_CUBIC);
currFace.face = currFace.faceMat.clone();
currFace.x1 = x1;
currFace.y1 = y1;
currFace.x2 = x2;
currFace.y2 = y2;
croppedFaces.push_back(currFace);
}
frame.release();
}
bool ANSFRHelper::LoadConfigFile(std::string configFile, FRConfig& config) {
try {
if (FileExist(configFile)) {
boost::property_tree::ptree pt;
std::vector <std::string> labels;
boost::property_tree::read_json(configFile, pt);
config._databasePath = GetData<std::string>(pt, "database_path");
config._recEngineFile = GetData<std::string>(pt, "rec_engine");
config._detEngineFile = GetData<std::string>(pt, "det_engine");
return true;
}
else return false;
}
catch (std::exception& e) {
std::cerr << "ANSFRHelper::LoadConfigFile" << e.what();
return false;
}
}
void ANSFRHelper::GetFilePaths(std::string rootPath, std::vector<struct Paths>& paths) {
/*
imagesPath--|
|--class0--|
| |--f0.jpg
| |--f1.jpg
|
|--class1--|
|--f0.jpg
|--f1.jpg
...
*/
DIR* dir;
struct dirent* entry;
std::string postfix = ".jpg";
if ((dir = opendir(rootPath.c_str())) != NULL) {
while ((entry = readdir(dir)) != NULL) {
std::string class_path = rootPath + "/" + entry->d_name;
DIR* class_dir = opendir(class_path.c_str());
struct dirent* file_entry;
while ((file_entry = readdir(class_dir)) != NULL) {
std::string name(file_entry->d_name);
if (name.length() >= postfix.length() && 0 == name.compare(name.length() - postfix.length(), postfix.length(), postfix))
if (file_entry->d_type != DT_DIR) {
struct Paths tempPaths;
tempPaths.className = std::string(entry->d_name);
tempPaths.absPath = class_path + "/" + name;
paths.push_back(tempPaths);
}
}
}
closedir(dir);
}
}
unsigned char* ANSCENTER::ANSFRHelper::CVMatToBytes(cv::Mat image, unsigned int& bufferLengh)
{
int size = int(image.total() * image.elemSize());
unsigned char* bytes = new unsigned char[size]; // you will have to delete[] that later
std::memcpy(bytes, image.data, size * sizeof(unsigned char));
bufferLengh = size * sizeof(unsigned char);
return bytes;
}
std::string ANSFRHelper::StringCurrentDateTime()
{
// Get the current time
std::time_t t = std::time(nullptr);
std::tm now = {}; // Initialize to all zeros
localtime_s(&now, &t);
// Convert the time to a string
std::stringstream ss;
ss << std::put_time(&now, "%Y%m%d%H%M%S");
std::string currentDateTime = ss.str();
return currentDateTime;
}
std::string ANSFRHelper::FaceObjectsToJsonString(const std::vector<FaceResultObject>& faces) {
boost::property_tree::ptree root;
boost::property_tree::ptree detectedObjects;
for (int i = 0; i < faces.size(); i++) {
boost::property_tree::ptree detectedNode;
detectedNode.put("user_id", faces[i].userId);
detectedNode.put("user_name", faces[i].userName);
detectedNode.put("similarity", faces[i].similarity);
detectedNode.put("is_unknown", faces[i].isUnknown);
detectedNode.put("prob", faces[i].confidence);
detectedNode.put("x", faces[i].box.x);
detectedNode.put("y", faces[i].box.y);
detectedNode.put("width", faces[i].box.width);
detectedNode.put("height", faces[i].box.height);
detectedNode.put("extra_info", faces[i].extraInformation);
// we might add masks into this using comma seperated string
detectedObjects.push_back(std::make_pair("", detectedNode));
}
root.add_child("results", detectedObjects);
std::ostringstream stream;
boost::property_tree::write_json(stream, root,false);
std::string faceResult = stream.str();
return faceResult;
}
std::string ANSFRHelper::FaceAttributeToJsonString(Face face) {
boost::property_tree::ptree root;
boost::property_tree::ptree detectedObjects;
boost::property_tree::ptree detectedNode;
if(face.isAntispoofingEnabled()) {
if (face.isReal()) {
detectedNode.put("Antispoofing", "Real");
}
else {
detectedNode.put("Antispoofing", "Fake");
}
}
if (face.isFaceLivenessEnabled()) {
if (face.getFaceLiveness() == 1) {
detectedNode.put("FaceLiveness", "Real");
}
else {
detectedNode.put("FaceLiveness", "Fake");
}
}
if (face.isAgeGenderEnabled()) {
if (face.isMale()) {
detectedNode.put("Gender", "Male");
}
else {
detectedNode.put("Gender", "Female");
}
int age = face.getAge();
if (age < 12)
detectedNode.put("Age", "Kids");
else if (age < 20)
detectedNode.put("Age", "Young Adults");
else if (age < 60)
detectedNode.put("Age", "Adults");
else
detectedNode.put("Age", "Elders");
// Numeric age alongside the bucket — additive key, existing
// consumers keep parsing "Age" unchanged.
detectedNode.put("AgeYears", std::to_string(age));
}
if (face.isEmotionsEnabled()) {
const auto& [emotionName, emotionConfidence] = face.getMainEmotion();
std::ostringstream emotionStr;
// The v1.4 ONNX classifier's classes (emotionsrecognition.names:
// Angry, Fear, Happy, Neutral, Sad) are reported VERBATIM. Only
// the legacy OpenVINO model's lowercase classes keep the
// historical display remap so v1.1 output is unchanged.
std::string adjustedEmotionName = emotionName;
if (emotionName == "sad")
adjustedEmotionName = "Attentive";
else if (emotionName == "happy")
adjustedEmotionName = "Happy";
else if (emotionName == "neutral")
adjustedEmotionName = "Neutral";
else if (emotionName == "anger")
adjustedEmotionName = "Attentive";
else if (emotionName == "surprise")
adjustedEmotionName = "Attentive";
emotionStr << adjustedEmotionName << " (" << static_cast<int>(emotionConfidence * 100) << "%)";
detectedNode.put("Emotion", emotionStr.str());
}
if (face.isHeadPoseEnabled()) {
const auto& headPose = face.getHeadPose();
std::ostringstream headPoseStr;
headPoseStr << "(" << static_cast<int>(headPose.angle_r) << ", "
<< static_cast<int>(headPose.angle_p) << ", "
<< static_cast<int>(headPose.angle_y) << ")";
detectedNode.put("Pose(r,p,y)", headPoseStr.str());
}
// we might add masks into this using comma seperated string
detectedObjects.push_back(std::make_pair("", detectedNode));
root.add_child("attributes", detectedObjects);
std::ostringstream stream;
boost::property_tree::write_json(stream, root,false);
std::string faceResult = stream.str();
return faceResult;
}
// Encode non-ASCII UTF-8 characters as double-escaped Unicode (\uXXXX) for JSON transport.
// Surrogate pairs are used for codepoints above U+FFFF.
static std::string DoubleEscapeUnicode(const std::string& utf8Str) {
bool hasNonAscii = false;
for (unsigned char c : utf8Str) {
if (c >= 0x80) { hasNonAscii = true; break; }
}
if (!hasNonAscii) return utf8Str;
std::string result;
result.reserve(utf8Str.size() * 2);
size_t i = 0;
while (i < utf8Str.size()) {
unsigned char c = static_cast<unsigned char>(utf8Str[i]);
if (c < 0x80) { result += utf8Str[i++]; continue; }
uint32_t cp = 0;
if ((c & 0xE0) == 0xC0 && i + 1 < utf8Str.size()) {
cp = ((c & 0x1F) << 6) | (static_cast<unsigned char>(utf8Str[i + 1]) & 0x3F); i += 2;
} else if ((c & 0xF0) == 0xE0 && i + 2 < utf8Str.size()) {
cp = ((c & 0x0F) << 12) | ((static_cast<unsigned char>(utf8Str[i + 1]) & 0x3F) << 6) | (static_cast<unsigned char>(utf8Str[i + 2]) & 0x3F); i += 3;
} else if ((c & 0xF8) == 0xF0 && i + 3 < utf8Str.size()) {
cp = ((c & 0x07) << 18) | ((static_cast<unsigned char>(utf8Str[i + 1]) & 0x3F) << 12) | ((static_cast<unsigned char>(utf8Str[i + 2]) & 0x3F) << 6) | (static_cast<unsigned char>(utf8Str[i + 3]) & 0x3F); i += 4;
} else { i++; continue; }
if (cp <= 0xFFFF) { char buf[8]; snprintf(buf, sizeof(buf), "\\u%04x", cp); result += buf; }
else { cp -= 0x10000; char buf[16]; snprintf(buf, sizeof(buf), "\\u%04x\\u%04x", 0xD800 + (uint16_t)(cp >> 10), 0xDC00 + (uint16_t)(cp & 0x3FF)); result += buf; }
}
return result;
}
// Using nlohmann/json (consistent with FaceObjectsToJsonString)
std::string ANSFRHelper::UserRecordToJsonString(const UserRecord userRecord) {
try {
nlohmann::json faceIdArray = nlohmann::json::array();
for (const auto& faceId : userRecord.FaceIds) {
faceIdArray.push_back(faceId);
}
nlohmann::json root = {
{"user_id", std::to_string(userRecord.UserId)},
{"user_code", userRecord.UserCode},
{"user_username", DoubleEscapeUnicode(userRecord.UserName)},
{"face_ids", faceIdArray}
};
return root.dump();
}
catch (const std::exception&) {
return R"({})";
}
}
// Using nlohmann/json (consistent with FaceObjectsToJsonString)
std::string ANSFRHelper::UserRecordsToJsonString(const std::vector<UserRecord> userRecords) {
try {
nlohmann::json root;
auto& records = root["user_records"] = nlohmann::json::array();
for (const auto& userRecord : userRecords) {
nlohmann::json faceIdArray = nlohmann::json::array();
for (const auto& faceId : userRecord.FaceIds) {
faceIdArray.push_back(faceId);
}
records.push_back({
{"user_id", std::to_string(userRecord.UserId)},
{"user_code", userRecord.UserCode},
{"user_username", DoubleEscapeUnicode(userRecord.UserName)},
{"face_ids", faceIdArray}
});
}
return root.dump();
}
catch (const std::exception&) {
return R"({"user_records":[]})";
}
}
std::string ANSFRHelper::FaceRecordToJsonString(const FaceRecord faceRecord) {
boost::property_tree::ptree root;
root.put("face_id", faceRecord.FaceId);
root.put("user_id", faceRecord.UserId);
root.put("image_path", faceRecord.ImagePath);
std::ostringstream stream;
boost::property_tree::write_json(stream, root,false);
std::string userRecordResult = stream.str();
return userRecordResult;
}
std::string ANSFRHelper::FaceRecordsToJsonString(const std::vector<FaceRecord> faceRecords) {
boost::property_tree::ptree root;
boost::property_tree::ptree faceNodes;
for (int i = 0; i < faceRecords.size(); i++) {
FaceRecord faceRecord = faceRecords[i];
boost::property_tree::ptree faceNode;
faceNode.put("face_id", faceRecord.FaceId);
faceNode.put("user_id", faceRecord.UserId);
faceNode.put("image_path", faceRecord.ImagePath);
faceNodes.push_back(std::make_pair("", faceNode));
}
root.add_child("face_records", faceNodes);
std::ostringstream stream;
boost::property_tree::write_json(stream, root,false);
std::string userRecordResult = stream.str();
return userRecordResult;
}
cv::Mat ANSFRHelper::PreprocessImg(cv::Mat& img, int input_w, int input_h) {
try {
int w, h, x, y;
float r_w = input_w / (img.cols * 1.0);
float r_h = input_h / (img.rows * 1.0);
if (r_h > r_w) {
w = input_w;
h = r_w * img.rows;
x = 0;
y = (input_h - h) / 2;
}
else {
w = r_h * img.cols;
h = input_h;
x = (input_w - w) / 2;
y = 0;
}
cv::Mat re(h, w, CV_8UC3);
cv::resize(img, re, re.size(), 0, 0, cv::INTER_LINEAR);
cv::Mat out(input_h, input_w, CV_8UC3, cv::Scalar(128, 128, 128));
re.copyTo(out(cv::Rect(x, y, re.cols, re.rows)));
return out;
}
catch (std::exception& e) {
cv::Mat result;
std::cerr << "ANSFRHelper::PreprocessImg" << e.what();
return result;
}
}
int ANSFRHelper::ReadFilesInDir(const char* p_dir_name, std::vector<std::string>& file_names) {
try {
DIR* p_dir = opendir(p_dir_name);
if (p_dir == nullptr) {
return -1;
}
struct dirent* p_file = nullptr;
while ((p_file = readdir(p_dir)) != nullptr) {
if (strcmp(p_file->d_name, ".") != 0 &&
strcmp(p_file->d_name, "..") != 0) {
std::string cur_file_name(p_file->d_name);
file_names.push_back(cur_file_name);
}
}
closedir(p_dir);
return 0;
}
catch (std::exception& e) {
std::cerr << "ANSFRHelper::ReadFilesInDir" << e.what();
return -1;
}
}
cv::Rect ANSFRHelper::GetRectAdaptLandmark(cv::Mat& img, int input_w, int input_h, float bbox[4], float lmk[10]) {
int l, r, t, b;
float r_w = input_w / (img.cols * 1.0);
float r_h = input_h / (img.rows * 1.0);
if (r_h > r_w) {
l = bbox[0] / r_w;
r = bbox[2] / r_w;
t = (bbox[1] - (input_h - r_w * img.rows) / 2) / r_w;
b = (bbox[3] - (input_h - r_w * img.rows) / 2) / r_w;
for (int i = 0; i < 10; i += 2) {
lmk[i] /= r_w;
lmk[i + 1] = (lmk[i + 1] - (input_h - r_w * img.rows) / 2) / r_w;
}
}
else {
l = (bbox[0] - (input_w - r_h * img.cols) / 2) / r_h;
r = (bbox[2] - (input_w - r_h * img.cols) / 2) / r_h;
t = bbox[1] / r_h;
b = bbox[3] / r_h;
for (int i = 0; i < 10; i += 2) {
lmk[i] = (lmk[i] - (input_w - r_h * img.cols) / 2) / r_h;
lmk[i + 1] /= r_h;
}
}
return cv::Rect(l, t, r - l, b - t);
}
float ANSFRHelper::IOU(float lbox[4], float rbox[4]) {
float interBox[] = {
MAX(lbox[0], rbox[0]), //left
MIN(lbox[2], rbox[2]), //right
MAX(lbox[1], rbox[1]), //top
MIN(lbox[3], rbox[3]), //bottom
};
if (interBox[2] > interBox[3] || interBox[0] > interBox[1])
return 0.0f;
float interBoxS = (interBox[1] - interBox[0]) * (interBox[3] - interBox[2]);
return interBoxS / ((lbox[2] - lbox[0]) * (lbox[3] - lbox[1]) + (rbox[2] - rbox[0]) * (rbox[3] - rbox[1]) - interBoxS + 0.000001f);
}
bool ANSFRHelper::CMP(const Detection& a, const Detection& b) {
return a.class_confidence > b.class_confidence;
}
void ANSFRHelper::NMS(std::vector<Detection>& res, float* output, float nms_thresh) {
std::vector<Detection> dets;
for (int i = 0; i < output[0]; i++) {
if (output[15 * i + 1 + 4] <= 0.1) continue;
Detection det;
memcpy(&det, &output[15 * i + 1], sizeof(Detection));
dets.push_back(det);
}
std::sort(dets.begin(), dets.end(), ANSFRHelper::CMP);
for (size_t m = 0; m < dets.size(); ++m) {
auto& item = dets[m];
res.push_back(item);
//std::cout << item.class_confidence << " bbox " << item.bbox[0] << ", " << item.bbox[1] << ", " << item.bbox[2] << ", " << item.bbox[3] << std::endl;
for (size_t n = m + 1; n < dets.size(); ++n) {
if (ANSFRHelper::IOU(item.bbox, dets[n].bbox) > nms_thresh) {
dets.erase(dets.begin() + n);
--n;
}
}
}
}
std::map<std::string, Weights> ANSFRHelper::LoadWeights(const std::string file) {
std::cout << "Loading weights: " << file << std::endl;
std::map<std::string, Weights> weightMap;
// Open weights file
std::ifstream input(file);
assert(input.is_open() && "Unable to load weight file.");
// Read number of weight blobs
int32_t count;
input >> count;
assert(count > 0 && "Invalid weight map file.");
while (count--)
{
Weights wt{ DataType::kFLOAT, nullptr, 0 };
uint32_t size;
// Read name and type of blob
std::string name;
input >> name >> std::dec >> size;
wt.type = DataType::kFLOAT;
// Load blob
uint32_t* val = reinterpret_cast<uint32_t*>(malloc(sizeof(val) * size));
for (uint32_t x = 0, y = size; x < y; ++x)
{
input >> std::hex >> val[x];
}
wt.values = val;
wt.count = size;
weightMap[name] = wt;
}
return weightMap;
}
Weights ANSFRHelper::GetWeights(std::map<std::string, Weights>& weightMap, std::string key) {
if (weightMap.count(key) != 1) {
std::cerr << key << " not existed in weight map, fatal error!!!" << std::endl;
exit(-1);
}
return weightMap[key];
}
IScaleLayer* ANSFRHelper::AddBatchNorm2D(INetworkDefinition* network, std::map<std::string, Weights>& weightMap, ITensor& input, std::string lname, float eps) {
float* gamma = (float*)weightMap[lname + ".weight"].values;
float* beta = (float*)weightMap[lname + ".bias"].values;
float* mean = (float*)weightMap[lname + ".running_mean"].values;
float* var = (float*)weightMap[lname + ".running_var"].values;
int len = weightMap[lname + ".running_var"].count;
float* scval = reinterpret_cast<float*>(malloc(sizeof(float) * len));
for (int i = 0; i < len; i++) {
scval[i] = gamma[i] / sqrt(var[i] + eps);
}
Weights scale{ DataType::kFLOAT, scval, len };
float* shval = reinterpret_cast<float*>(malloc(sizeof(float) * len));
for (int i = 0; i < len; i++) {
shval[i] = beta[i] - mean[i] * gamma[i] / sqrt(var[i] + eps);
}
Weights shift{ DataType::kFLOAT, shval, len };
float* pval = reinterpret_cast<float*>(malloc(sizeof(float) * len));
for (int i = 0; i < len; i++) {
pval[i] = 1.0;
}
Weights power{ DataType::kFLOAT, pval, len };
weightMap[lname + ".scale"] = scale;
weightMap[lname + ".shift"] = shift;
weightMap[lname + ".power"] = power;
IScaleLayer* scale_1 = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power);
assert(scale_1);
return scale_1;
}
/// <summary>
/// MatMul
/// </summary>
//MatMul::MatMul() {
// try {
// ANSFRHelper::ANSFRHelper::CheckCudaStatus(cudaMalloc(&workspace, workspaceSize));
// ANSFRHelper::CheckCublasStatus(cublasLtCreate(&ltHandle));
// ANSFRHelper::CheckCudaStatus(cudaStreamCreate(&stream));
// }
// catch (std::exception& e) {
// std::cerr << "MatMul::MatMul" << e.what();
// }
//}
//void MatMul::Init(float* knownEmbeds, int numRow, int numCol) {
// try {
// m = static_cast<const int>(numRow);
// k = static_cast<const int>(numCol);
// lda = static_cast<const int>(numCol);
// ldb = static_cast<const int>(numCol);
// ldc = static_cast<const int>(numRow);
// // alloc and copy known embeddings to GPU
// ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast<void**>(&dA), m * k * sizeof(float)));
// ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(dA, knownEmbeds, m * k * sizeof(float), cudaMemcpyHostToDevice, stream));
// // create operation desciriptor; see cublasLtMatmulDescAttributes_t for details about defaults;
// // here we just need to set the transforms for A and B
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescCreate(&operationDesc, computeType, cudaDataType));
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)));
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transb)));
// // create matrix descriptors, we are good with the details here so no need to set any extra attributes
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Adesc, cudaDataType, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda));
// // create preference handle; here we could use extra attributes to disable tensor ops or to make sure algo selected
// // will work with badly aligned A, B, C; here for simplicity we just assume A,B,C are always well aligned (e.g.
// // directly come from cudaMalloc)
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceCreate(&preference));
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceSetAttribute(preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)));
// }
// catch (std::exception& e) {
// std::cerr << "MatMul::Init" << e.what();
// }
//}
//void MatMul::Calculate(float* embeds, int embedCount, float* outputs) {
// try {
// n = embedCount;
// // Allocate arrays on GPU
// ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast<void**>(&dB), k * n * sizeof(float)));
// ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast<void**>(&dC), m * n * sizeof(float)));
// ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(dB, embeds, k * n * sizeof(float), cudaMemcpyHostToDevice, stream));
// // create matrix descriptors, we are good with the details here so no need to set any extra attributes
// cublasLtMatrixLayout_t Bdesc = NULL, Cdesc = NULL;
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Bdesc, cudaDataType, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb));
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Cdesc, cudaDataType, m, n, ldc));
// // we just need the best available heuristic to try and run matmul. There is no guarantee this will work, e.g. if A
// // is badly aligned, you can request more (e.g. 32) algos and try to run them one by one until something works
// int returnedResults = 0;
// cublasLtMatmulHeuristicResult_t heuristicResult = {};
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulAlgoGetHeuristic(ltHandle, operationDesc, Adesc, Bdesc, Cdesc, Cdesc, preference, 1, &heuristicResult, &returnedResults));
// if (returnedResults == 0) {
// ANSFRHelper::CheckCublasStatus(CUBLAS_STATUS_NOT_SUPPORTED);
// }
// // Do the actual multiplication
// ANSFRHelper::CheckCublasStatus(cublasLtMatmul(ltHandle, operationDesc, &alpha, dA, Adesc, dB, Bdesc, &beta, dC, Cdesc, dC, Cdesc, &heuristicResult.algo, workspace,
// workspaceSize, stream));
// // Cleanup: descriptors are no longer needed as all GPU work was already enqueued
// if (Cdesc)
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Cdesc));
// if (Bdesc)
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Bdesc));
// // Copy the result on host memory
// ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(outputs, dC, m * n * sizeof(float), cudaMemcpyDeviceToHost, stream));
// // CUDA stream sync
// ANSFRHelper::CheckCudaStatus(cudaStreamSynchronize(stream));
// // Free GPU memory
// ANSFRHelper::CheckCudaStatus(cudaFree(dB));
// ANSFRHelper::CheckCudaStatus(cudaFree(dC));
// }
// catch (std::exception& e) {
// std::cerr <<"MatMul::Calculate"<< e.what();
// }
//}
//MatMul::~MatMul() {
// try {
// if (preference)
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceDestroy(preference));
// if (Adesc)
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Adesc));
// if (operationDesc)
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescDestroy(operationDesc));
// ANSFRHelper::CheckCublasStatus(cublasLtDestroy(ltHandle));
// ANSFRHelper::CheckCudaStatus(cudaFree(dA));
// ANSFRHelper::CheckCudaStatus(cudaFree(workspace));
// ANSFRHelper::CheckCudaStatus(cudaStreamDestroy(stream));
// }
// catch (std::exception& e) {
// std::cout << "MatMul::~MatMul" << e.what();
// }
//}
//Int8EntropyCalibrator2::Int8EntropyCalibrator2(int batchsize, int input_w, int input_h, const char* img_dir, const char* calib_table_name, const char* input_blob_name, bool read_cache)
// : batchsize_(batchsize)
// , input_w_(input_w)
// , input_h_(input_h)
// , img_idx_(0)
// , img_dir_(img_dir)
// , calib_table_name_(calib_table_name)
// , input_blob_name_(input_blob_name)
// , read_cache_(read_cache)
//{
// input_count_ = 3 * input_w * input_h * batchsize;
// CUDACHECK(cudaMalloc(&device_input_, input_count_ * sizeof(float)));
// ANSFRHelper::ReadFilesInDir(img_dir, img_files_);
//}
//Int8EntropyCalibrator2::~Int8EntropyCalibrator2()
//{
// CUDACHECK(cudaFree(device_input_));
//}
//int Int8EntropyCalibrator2::getBatchSize() const TRT_NOEXCEPT
//{
// return batchsize_;
//}
//bool Int8EntropyCalibrator2::getBatch(void* bindings[], const char* names[], int nbBindings) TRT_NOEXCEPT
//{
// if (img_idx_ + batchsize_ > (int)img_files_.size()) {
// return false;
// }
// std::vector<cv::Mat> input_imgs_;
// for (int i = img_idx_; i < img_idx_ + batchsize_; i++) {
// std::cout << img_files_[i] << " " << i << std::endl;
// cv::Mat temp = cv::imread(img_dir_ + img_files_[i]);
// if (temp.empty()) {
// std::cerr << "Fatal error: image cannot open!" << std::endl;
// return false;
// }
// cv::Mat pr_img = ANSFRHelper::PreprocessImg(temp, input_w_, input_h_);
// input_imgs_.push_back(pr_img);
// }
// img_idx_ += batchsize_;
// cv::Mat blob = cv::dnn::blobFromImages(input_imgs_, 1.0, cv::Size(input_w_, input_h_), cv::Scalar(104, 117, 123), false, false);
// CUDACHECK(cudaMemcpy(device_input_, blob.ptr<float>(0), input_count_ * sizeof(float), cudaMemcpyHostToDevice));
// assert(!strcmp(names[0], input_blob_name_));
// bindings[0] = device_input_;
// return true;
//}
//const void* Int8EntropyCalibrator2::readCalibrationCache(size_t& length) TRT_NOEXCEPT
//{
// std::cout << "reading calib cache: " << calib_table_name_ << std::endl;
// calib_cache_.clear();
// std::ifstream input(calib_table_name_, std::ios::binary);
// input >> std::noskipws;
// if (read_cache_ && input.good())
// {
// std::copy(std::istream_iterator<char>(input), std::istream_iterator<char>(), std::back_inserter(calib_cache_));
// }
// length = calib_cache_.size();
// return length ? calib_cache_.data() : nullptr;
//}
//void Int8EntropyCalibrator2::writeCalibrationCache(const void* cache, size_t length) TRT_NOEXCEPT
//{
// std::cout << "writing calib cache: " << calib_table_name_ << " size: " << length << std::endl;
// std::ofstream output(calib_table_name_, std::ios::binary);
// output.write(reinterpret_cast<const char*>(cache), length);
//}
}