1134 lines
45 KiB
C++
1134 lines
45 KiB
C++
|
#include "ANSFRCommon.h"
|
||
|
|
#include "dirent.h"
|
||
|
|
#include <ctime>
|
||
|
|
#include <opencv2/imgcodecs.hpp>
|
||
|
|
#include <opencv2/dnn/dnn.hpp>
|
||
|
|
#include <opencv2/highgui.hpp>
|
||
|
|
#include <utils/ocv_common.hpp>
|
||
|
|
namespace ANSCENTER {
|
||
|
|
constexpr size_t ndetections = 200;
|
||
|
|
template <typename T> T GetData(const boost::property_tree::ptree& pt, const std::string& key)
|
||
|
|
{
|
||
|
|
T ret;
|
||
|
|
if (boost::optional<T> data = pt.get_optional<T>(key))
|
||
|
|
{
|
||
|
|
ret = data.get();
|
||
|
|
}
|
||
|
|
|
||
|
|
return ret;
|
||
|
|
}
|
||
|
|
|
||
|
|
Face::Face(size_t id, cv::Rect& location) :
|
||
|
|
_location(location), _intensity_mean(0.f), _id(id), _age(-1),
|
||
|
|
_maleScore(0), _femaleScore(0), _realFaceConfidence(0),
|
||
|
|
_isAgeGenderEnabled(false), _isEmotionsEnabled(false),
|
||
|
|
_isHeadPoseEnabled(false), _isLandmarksEnabled(false), _isAntispoofingEnabled(false)
|
||
|
|
{
|
||
|
|
HeadPoseResults headPose;
|
||
|
|
headPose.angle_p = 0.f;
|
||
|
|
headPose.angle_r = 0.f;
|
||
|
|
headPose.angle_y = 0.f;
|
||
|
|
_headPose = headPose;
|
||
|
|
}
|
||
|
|
|
||
|
|
void Face::updateAge(float value) {
|
||
|
|
_age = (_age == -1) ? value : 0.95f * _age + 0.05f * value;
|
||
|
|
}
|
||
|
|
|
||
|
|
void Face::updateGender(float value) {
|
||
|
|
if (value < 0)
|
||
|
|
return;
|
||
|
|
|
||
|
|
if (value > 0.5) {
|
||
|
|
_maleScore += value - 0.5f;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
_femaleScore += 0.5f - value;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
void Face::updateEmotions(std::map<std::string, float> values) {
|
||
|
|
for (auto& kv : values) {
|
||
|
|
if (_emotions.find(kv.first) == _emotions.end()) {
|
||
|
|
_emotions[kv.first] = kv.second;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
_emotions[kv.first] = 0.9f * _emotions[kv.first] + 0.1f * kv.second;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
void Face::updateHeadPose(HeadPoseResults values) {
|
||
|
|
_headPose = values;
|
||
|
|
}
|
||
|
|
|
||
|
|
void Face::updateLandmarks(std::vector<float> values) {
|
||
|
|
_landmarks = std::move(values);
|
||
|
|
}
|
||
|
|
|
||
|
|
void Face::updateRealFaceConfidence(float value) {
|
||
|
|
_realFaceConfidence = value;
|
||
|
|
}
|
||
|
|
void Face::updateFaceLiveness(int value) {
|
||
|
|
_faceLiveness = value;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
int Face::getAge() {
|
||
|
|
return static_cast<int>(std::floor(_age + 0.5f));
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Face::isMale() {
|
||
|
|
return _maleScore > _femaleScore;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Face::isReal() {
|
||
|
|
return _realFaceConfidence > 0.4;
|
||
|
|
}
|
||
|
|
float Face::getAntispoofingScore() {
|
||
|
|
return _realFaceConfidence;
|
||
|
|
}
|
||
|
|
int Face::getFaceLiveness() {
|
||
|
|
return _faceLiveness;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
std::map<std::string, float> Face::getEmotions() {
|
||
|
|
return _emotions;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::pair<std::string, float> Face::getMainEmotion() {
|
||
|
|
auto x = std::max_element(_emotions.begin(), _emotions.end(),
|
||
|
|
[](const std::pair<std::string, float>& p1, const std::pair<std::string, float>& p2) {
|
||
|
|
return p1.second < p2.second; });
|
||
|
|
|
||
|
|
return std::make_pair(x->first, x->second);
|
||
|
|
}
|
||
|
|
|
||
|
|
HeadPoseResults Face::getHeadPose() {
|
||
|
|
return _headPose;
|
||
|
|
}
|
||
|
|
|
||
|
|
const std::vector<float>& Face::getLandmarks() {
|
||
|
|
return _landmarks;
|
||
|
|
}
|
||
|
|
|
||
|
|
size_t Face::getId() {
|
||
|
|
return _id;
|
||
|
|
}
|
||
|
|
|
||
|
|
void Face::ageGenderEnable(bool value) {
|
||
|
|
_isAgeGenderEnabled = value;
|
||
|
|
}
|
||
|
|
void Face::emotionsEnable(bool value) {
|
||
|
|
_isEmotionsEnabled = value;
|
||
|
|
}
|
||
|
|
void Face::headPoseEnable(bool value) {
|
||
|
|
_isHeadPoseEnabled = value;
|
||
|
|
}
|
||
|
|
void Face::landmarksEnable(bool value) {
|
||
|
|
_isLandmarksEnabled = value;
|
||
|
|
}
|
||
|
|
void Face::antispoofingEnable(bool value) {
|
||
|
|
_isAntispoofingEnabled = value;
|
||
|
|
}
|
||
|
|
void Face::faceLivenessEnable(bool value) {
|
||
|
|
_isFaceLivenessEnabled = value;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
bool Face::isAgeGenderEnabled() {
|
||
|
|
return _isAgeGenderEnabled;
|
||
|
|
}
|
||
|
|
bool Face::isEmotionsEnabled() {
|
||
|
|
return _isEmotionsEnabled;
|
||
|
|
}
|
||
|
|
bool Face::isHeadPoseEnabled() {
|
||
|
|
return _isHeadPoseEnabled;
|
||
|
|
}
|
||
|
|
bool Face::isLandmarksEnabled() {
|
||
|
|
return _isLandmarksEnabled;
|
||
|
|
}
|
||
|
|
bool Face::isAntispoofingEnabled() {
|
||
|
|
return _isAntispoofingEnabled;
|
||
|
|
}
|
||
|
|
bool Face::isFaceLivenessEnabled() {
|
||
|
|
return _isFaceLivenessEnabled;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
float calcIoU(cv::Rect& src, cv::Rect& dst) {
|
||
|
|
cv::Rect i = src & dst;
|
||
|
|
cv::Rect u = src | dst;
|
||
|
|
|
||
|
|
return static_cast<float>(i.area()) / static_cast<float>(u.area());
|
||
|
|
}
|
||
|
|
|
||
|
|
float calcMean(const cv::Mat& src) {
|
||
|
|
cv::Mat tmp;
|
||
|
|
cv::cvtColor(src, tmp, cv::COLOR_BGR2GRAY);
|
||
|
|
cv::Scalar mean = cv::mean(tmp);
|
||
|
|
|
||
|
|
return static_cast<float>(mean[0]);
|
||
|
|
}
|
||
|
|
|
||
|
|
Face matchFace(cv::Rect rect, std::list<Face>& faces) {
|
||
|
|
Face face(0,rect);
|
||
|
|
float maxIoU = 0.55f;
|
||
|
|
for (auto&& f : faces) {
|
||
|
|
float iou = calcIoU(rect, f._location);
|
||
|
|
if (iou > maxIoU) {
|
||
|
|
face = f;
|
||
|
|
maxIoU = iou;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return face;
|
||
|
|
}
|
||
|
|
BaseDetection::BaseDetection(const std::string& pathToModel, bool doRawOutputMessages)
|
||
|
|
: pathToModel(pathToModel), doRawOutputMessages(doRawOutputMessages) {
|
||
|
|
}
|
||
|
|
bool BaseDetection::enabled() const {
|
||
|
|
return bool(request);
|
||
|
|
}
|
||
|
|
|
||
|
|
AntispoofingClassifier::AntispoofingClassifier(const std::string& pathToModel, bool doRawOutputMessages)
|
||
|
|
: BaseDetection(pathToModel, doRawOutputMessages)
|
||
|
|
{
|
||
|
|
_modelFilePath = pathToModel;
|
||
|
|
}
|
||
|
|
float AntispoofingClassifier::runInfer(const cv::Mat& frame) {
|
||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||
|
|
try {
|
||
|
|
cv::Mat face = frame.clone();
|
||
|
|
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
|
||
|
|
resize2tensor(face, inSlice);
|
||
|
|
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
|
||
|
|
request.infer();
|
||
|
|
face.release();
|
||
|
|
float r = request.get_output_tensor().data<const float>()[0] * 100;
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
catch (const std::exception& error) {
|
||
|
|
std::cerr << "Error: " << error.what() << std::endl;
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
std::shared_ptr<ov::Model> AntispoofingClassifier::read(const ov::Core& core) {
|
||
|
|
slog::info << "Reading model: " << _modelFilePath << slog::endl;
|
||
|
|
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
|
||
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
||
|
|
ppp.input().tensor().
|
||
|
|
set_element_type(ov::element::u8).
|
||
|
|
set_layout("NHWC");
|
||
|
|
ppp.input().preprocess().convert_layout("NCHW");
|
||
|
|
ppp.output().tensor().set_element_type(ov::element::f32);
|
||
|
|
model = ppp.build();
|
||
|
|
inShape = model->input().get_shape();
|
||
|
|
inShape[0] = ndetections;
|
||
|
|
//ov::set_batch(model, { 1, int64_t(ndetections) });
|
||
|
|
ov::set_batch(model, { 1 }); // Force single batch for all inputs
|
||
|
|
|
||
|
|
return model;
|
||
|
|
}
|
||
|
|
|
||
|
|
AgeGenderDetection::AgeGenderDetection(const std::string& pathToModel,
|
||
|
|
bool doRawOutputMessages)
|
||
|
|
: BaseDetection(pathToModel, doRawOutputMessages)
|
||
|
|
{
|
||
|
|
_modelFilePath = pathToModel;
|
||
|
|
}
|
||
|
|
AgeGenderResults AgeGenderDetection::runInfer(const cv::Mat& frame) {
|
||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||
|
|
try {
|
||
|
|
cv::Mat face = frame.clone();
|
||
|
|
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
|
||
|
|
resize2tensor(face, inSlice);
|
||
|
|
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
|
||
|
|
request.infer();
|
||
|
|
face.release();
|
||
|
|
AgeGenderResults r = { request.get_tensor(outputAge).data<const float>()[0] * 100,
|
||
|
|
request.get_tensor(outputGender).data<const float>()[0 * 2 + 1] };
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
catch (const std::exception& error) {
|
||
|
|
AgeGenderResults r = { -1, -1 };
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
std::shared_ptr<ov::Model> AgeGenderDetection::read(const ov::Core& core) {
|
||
|
|
slog::info << "Reading model: " << _modelFilePath << slog::endl;
|
||
|
|
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
|
||
|
|
outputAge = "age_conv3";
|
||
|
|
outputGender = "prob";
|
||
|
|
|
||
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
||
|
|
ppp.input().tensor().
|
||
|
|
set_element_type(ov::element::u8).
|
||
|
|
set_layout("NHWC");
|
||
|
|
ppp.input().preprocess().
|
||
|
|
convert_element_type(ov::element::f32).
|
||
|
|
convert_layout("NCHW");
|
||
|
|
ppp.output(outputAge).tensor().set_element_type(ov::element::f32);
|
||
|
|
ppp.output(outputGender).tensor().set_element_type(ov::element::f32);
|
||
|
|
model = ppp.build();
|
||
|
|
inShape = model->input().get_shape();
|
||
|
|
inShape[0] = ndetections;
|
||
|
|
//ov::set_batch(model, { 1, int64_t(ndetections) });
|
||
|
|
ov::set_batch(model, { 1 }); // Force single batch for all inputs
|
||
|
|
return model;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
HeadPoseDetection::HeadPoseDetection(const std::string& pathToModel,
|
||
|
|
bool doRawOutputMessages)
|
||
|
|
: BaseDetection(pathToModel, doRawOutputMessages),
|
||
|
|
outputAngleR("angle_r_fc"), outputAngleP("angle_p_fc"), outputAngleY("angle_y_fc")
|
||
|
|
{
|
||
|
|
_modelFilePath = pathToModel;
|
||
|
|
}
|
||
|
|
HeadPoseResults HeadPoseDetection::runInfer(const cv::Mat& frame) {
|
||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||
|
|
try {
|
||
|
|
cv::Mat face = frame.clone();
|
||
|
|
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
|
||
|
|
resize2tensor(face, inSlice);
|
||
|
|
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
|
||
|
|
request.infer();
|
||
|
|
face.release();
|
||
|
|
HeadPoseResults r = { request.get_tensor(outputAngleR).data<const float>()[0],
|
||
|
|
request.get_tensor(outputAngleP).data<const float>()[0],
|
||
|
|
request.get_tensor(outputAngleY).data<const float>()[0] };
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
catch (const std::exception& error) {
|
||
|
|
HeadPoseResults r = { -1, -1, -1 };
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
std::shared_ptr<ov::Model> HeadPoseDetection::read(const ov::Core& core) {
|
||
|
|
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
|
||
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
||
|
|
ppp.input().tensor().
|
||
|
|
set_element_type(ov::element::u8).
|
||
|
|
set_layout("NHWC");
|
||
|
|
ppp.input().preprocess().convert_layout("NCHW");
|
||
|
|
ppp.output(outputAngleR).tensor().set_element_type(ov::element::f32);
|
||
|
|
ppp.output(outputAngleP).tensor().set_element_type(ov::element::f32);
|
||
|
|
ppp.output(outputAngleY).tensor().set_element_type(ov::element::f32);
|
||
|
|
model = ppp.build();
|
||
|
|
inShape = model->input().get_shape();
|
||
|
|
inShape[0] = ndetections;
|
||
|
|
//ov::set_batch(model, { 1, int64_t(ndetections) });
|
||
|
|
ov::set_batch(model, { 1 }); // Force single batch for all inputs
|
||
|
|
return model;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
EmotionsDetection::EmotionsDetection(const std::string& pathToModel,
|
||
|
|
bool doRawOutputMessages)
|
||
|
|
: BaseDetection(pathToModel, doRawOutputMessages)
|
||
|
|
{
|
||
|
|
_modelFilePath = pathToModel;
|
||
|
|
}
|
||
|
|
std::map<std::string, float> EmotionsDetection::runInfer(const cv::Mat& frame) {
|
||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||
|
|
try {
|
||
|
|
cv::Mat face = frame.clone();
|
||
|
|
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
|
||
|
|
resize2tensor(face, inSlice);
|
||
|
|
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
|
||
|
|
request.infer();
|
||
|
|
face.release();
|
||
|
|
auto tensor = request.get_output_tensor();
|
||
|
|
auto emotionsVecSize = emotionsVec.size();
|
||
|
|
size_t numOfChannels = tensor.get_shape().at(1);
|
||
|
|
if (numOfChannels == emotionsVecSize) {
|
||
|
|
const float* emotionsValues = tensor.data<const float>();
|
||
|
|
auto outputIdxPos = emotionsValues + 0 * emotionsVecSize;
|
||
|
|
std::map<std::string, float> emotions;
|
||
|
|
for (size_t i = 0; i < emotionsVecSize; i++) {
|
||
|
|
emotions[emotionsVec[i]] = outputIdxPos[i];
|
||
|
|
}
|
||
|
|
return emotions;
|
||
|
|
}
|
||
|
|
else return { };
|
||
|
|
}
|
||
|
|
catch (const std::exception& error) {
|
||
|
|
std::cerr << "Error: " << error.what() << std::endl;
|
||
|
|
return { };
|
||
|
|
}
|
||
|
|
}
|
||
|
|
std::shared_ptr<ov::Model> EmotionsDetection::read(const ov::Core& core) {
|
||
|
|
slog::info << "Reading model: " << _modelFilePath << slog::endl;
|
||
|
|
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
|
||
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
||
|
|
ppp.input().tensor().
|
||
|
|
set_element_type(ov::element::u8).
|
||
|
|
set_layout("NHWC");
|
||
|
|
ppp.input().preprocess().convert_layout("NCHW");
|
||
|
|
ppp.output().tensor().set_element_type(ov::element::f32);
|
||
|
|
model = ppp.build();
|
||
|
|
inShape = model->input().get_shape();
|
||
|
|
inShape[0] = ndetections;
|
||
|
|
//ov::set_batch(model, { 1, int64_t(ndetections) });
|
||
|
|
ov::set_batch(model, { 1 }); // Force single batch for all inputs
|
||
|
|
return model;
|
||
|
|
}
|
||
|
|
|
||
|
|
FacialLandmarksDetection::FacialLandmarksDetection(const std::string& pathToModel,
|
||
|
|
bool doRawOutputMessages)
|
||
|
|
: BaseDetection(pathToModel, doRawOutputMessages)
|
||
|
|
{
|
||
|
|
_modelFilePath = pathToModel;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::vector<float> FacialLandmarksDetection::runInfer(const cv::Mat& frame) {
|
||
|
|
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||
|
|
std::vector<float> normedLandmarks;
|
||
|
|
try {
|
||
|
|
cv::Mat face = frame.clone();
|
||
|
|
auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} };
|
||
|
|
resize2tensor(face, inSlice);
|
||
|
|
request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} });
|
||
|
|
request.infer();
|
||
|
|
face.release();
|
||
|
|
auto tensor = request.get_output_tensor();
|
||
|
|
size_t n_lm = tensor.get_shape().at(1);
|
||
|
|
const float* normed_coordinates = tensor.data<const float>();
|
||
|
|
auto begin = 0;
|
||
|
|
auto end = begin + n_lm / 2;
|
||
|
|
for (auto i_lm = begin; i_lm < end; ++i_lm) {
|
||
|
|
float normed_x = normed_coordinates[2 * i_lm];
|
||
|
|
float normed_y = normed_coordinates[2 * i_lm + 1];
|
||
|
|
normedLandmarks.push_back(normed_x);
|
||
|
|
normedLandmarks.push_back(normed_y);
|
||
|
|
}
|
||
|
|
return normedLandmarks;
|
||
|
|
}
|
||
|
|
catch (const std::exception& error) {
|
||
|
|
std::cerr << "Error: " << error.what() << std::endl;
|
||
|
|
return normedLandmarks;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
std::shared_ptr<ov::Model> FacialLandmarksDetection::read(const ov::Core& core) {
|
||
|
|
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
|
||
|
|
ov::Shape outShape = model->output().get_shape();
|
||
|
|
if (outShape.size() != 2 && outShape.back() != 70) {
|
||
|
|
return model;
|
||
|
|
}
|
||
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
||
|
|
ppp.input().tensor().
|
||
|
|
set_element_type(ov::element::u8).
|
||
|
|
set_layout("NHWC");
|
||
|
|
ppp.input().preprocess().convert_layout("NCHW");
|
||
|
|
ppp.output().tensor().set_element_type(ov::element::f32);
|
||
|
|
model = ppp.build();
|
||
|
|
inShape = model->input().get_shape();
|
||
|
|
inShape[0] = ndetections;
|
||
|
|
//ov::set_batch(model, { 1, int64_t(ndetections) });
|
||
|
|
ov::set_batch(model, { 1 }); // Force single batch for all inputs
|
||
|
|
return model;
|
||
|
|
}
|
||
|
|
//void ANSFRHelper::ANSFRHelper::CheckCudaStatus(cudaError_t status) {
|
||
|
|
// try {
|
||
|
|
// if (status != cudaSuccess) {
|
||
|
|
// std::cerr << "CUDA API failed with status " << status << ": " << cudaGetErrorString(status) << std::endl;
|
||
|
|
// }
|
||
|
|
// }
|
||
|
|
// catch (std::exception& e) {
|
||
|
|
// std::cerr << "ANSFRHelper::CheckCublasStatus" << e.what();
|
||
|
|
// }
|
||
|
|
//}
|
||
|
|
|
||
|
|
//void ANSFRHelper::ANSFRHelper::CheckCublasStatus(cublasStatus_t status) {
|
||
|
|
// try {
|
||
|
|
// if (status != CUBLAS_STATUS_SUCCESS) {
|
||
|
|
// std::cerr << "cuBLAS API failed with status " << status << std::endl;
|
||
|
|
// }
|
||
|
|
// }
|
||
|
|
// catch (std::exception& e) {
|
||
|
|
// std::cerr << "ANSFRHelper::CheckCublasStatus" << e.what();
|
||
|
|
// }
|
||
|
|
//}
|
||
|
|
|
||
|
|
void ANSFRHelper::GetCroppedFaces(const cv::Mat &input, std::vector<Object>& outputBbox, int resize_w, int resize_h, std::vector<struct CroppedFace>& croppedFaces) {
|
||
|
|
croppedFaces.clear();
|
||
|
|
cv::Mat frame = input.clone();
|
||
|
|
if (outputBbox.size() > 0) {
|
||
|
|
for (std::vector<Object>::iterator it = outputBbox.begin(); it != outputBbox.end(); it++) {
|
||
|
|
int x1, y1, x2, y2;
|
||
|
|
x1 = (*it).box.x;
|
||
|
|
y1 = (*it).box.y;
|
||
|
|
x2 = (*it).box.x + (*it).box.width;
|
||
|
|
y2 = (*it).box.y + (*it).box.height;
|
||
|
|
|
||
|
|
cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2));
|
||
|
|
cv::Mat tempCrop = frame(facePos);
|
||
|
|
struct CroppedFace currFace;
|
||
|
|
cv::resize(tempCrop, currFace.faceMat, cv::Size(resize_h, resize_w), 0, 0, cv::INTER_CUBIC);
|
||
|
|
currFace.face = currFace.faceMat.clone();
|
||
|
|
currFace.x1 = x1;
|
||
|
|
currFace.y1 = y1;
|
||
|
|
currFace.x2 = x2;
|
||
|
|
currFace.y2 = y2;
|
||
|
|
croppedFaces.push_back(currFace);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// this will be the same image
|
||
|
|
int x1, y1, x2, y2;
|
||
|
|
x1 = 0;
|
||
|
|
y1 = 0;
|
||
|
|
x2 = frame.cols;
|
||
|
|
y2 = frame.rows;
|
||
|
|
cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2));
|
||
|
|
cv::Mat tempCrop = frame(facePos);
|
||
|
|
struct CroppedFace currFace;
|
||
|
|
cv::resize(tempCrop, currFace.faceMat, cv::Size(resize_h, resize_w), 0, 0, cv::INTER_CUBIC);
|
||
|
|
currFace.face = currFace.faceMat.clone();
|
||
|
|
currFace.x1 = x1;
|
||
|
|
currFace.y1 = y1;
|
||
|
|
currFace.x2 = x2;
|
||
|
|
currFace.y2 = y2;
|
||
|
|
croppedFaces.push_back(currFace);
|
||
|
|
}
|
||
|
|
frame.release();
|
||
|
|
|
||
|
|
}
|
||
|
|
bool ANSFRHelper::LoadConfigFile(std::string configFile, FRConfig& config) {
|
||
|
|
try {
|
||
|
|
if (FileExist(configFile)) {
|
||
|
|
boost::property_tree::ptree pt;
|
||
|
|
std::vector <std::string> labels;
|
||
|
|
boost::property_tree::read_json(configFile, pt);
|
||
|
|
|
||
|
|
config._databasePath = GetData<std::string>(pt, "database_path");
|
||
|
|
config._recEngineFile = GetData<std::string>(pt, "rec_engine");
|
||
|
|
config._detEngineFile = GetData<std::string>(pt, "det_engine");
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
else return false;
|
||
|
|
}
|
||
|
|
catch (std::exception& e) {
|
||
|
|
std::cerr << "ANSFRHelper::LoadConfigFile" << e.what();
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
void ANSFRHelper::GetFilePaths(std::string rootPath, std::vector<struct Paths>& paths) {
|
||
|
|
/*
|
||
|
|
imagesPath--|
|
||
|
|
|--class0--|
|
||
|
|
| |--f0.jpg
|
||
|
|
| |--f1.jpg
|
||
|
|
|
|
||
|
|
|--class1--|
|
||
|
|
|--f0.jpg
|
||
|
|
|--f1.jpg
|
||
|
|
...
|
||
|
|
*/
|
||
|
|
DIR* dir;
|
||
|
|
struct dirent* entry;
|
||
|
|
std::string postfix = ".jpg";
|
||
|
|
if ((dir = opendir(rootPath.c_str())) != NULL) {
|
||
|
|
while ((entry = readdir(dir)) != NULL) {
|
||
|
|
std::string class_path = rootPath + "/" + entry->d_name;
|
||
|
|
DIR* class_dir = opendir(class_path.c_str());
|
||
|
|
struct dirent* file_entry;
|
||
|
|
while ((file_entry = readdir(class_dir)) != NULL) {
|
||
|
|
std::string name(file_entry->d_name);
|
||
|
|
if (name.length() >= postfix.length() && 0 == name.compare(name.length() - postfix.length(), postfix.length(), postfix))
|
||
|
|
if (file_entry->d_type != DT_DIR) {
|
||
|
|
struct Paths tempPaths;
|
||
|
|
tempPaths.className = std::string(entry->d_name);
|
||
|
|
tempPaths.absPath = class_path + "/" + name;
|
||
|
|
paths.push_back(tempPaths);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
closedir(dir);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned char* ANSCENTER::ANSFRHelper::CVMatToBytes(cv::Mat image, unsigned int& bufferLengh)
|
||
|
|
{
|
||
|
|
int size = int(image.total() * image.elemSize());
|
||
|
|
unsigned char* bytes = new unsigned char[size]; // you will have to delete[] that later
|
||
|
|
std::memcpy(bytes, image.data, size * sizeof(unsigned char));
|
||
|
|
bufferLengh = size * sizeof(unsigned char);
|
||
|
|
return bytes;
|
||
|
|
}
|
||
|
|
std::string ANSFRHelper::StringCurrentDateTime()
|
||
|
|
{
|
||
|
|
// Get the current time
|
||
|
|
std::time_t t = std::time(nullptr);
|
||
|
|
std::tm now = {}; // Initialize to all zeros
|
||
|
|
localtime_s(&now, &t);
|
||
|
|
// Convert the time to a string
|
||
|
|
std::stringstream ss;
|
||
|
|
ss << std::put_time(&now, "%Y%m%d%H%M%S");
|
||
|
|
std::string currentDateTime = ss.str();
|
||
|
|
return currentDateTime;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string ANSFRHelper::FaceObjectsToJsonString(const std::vector<FaceResultObject>& faces) {
|
||
|
|
boost::property_tree::ptree root;
|
||
|
|
boost::property_tree::ptree detectedObjects;
|
||
|
|
for (int i = 0; i < faces.size(); i++) {
|
||
|
|
boost::property_tree::ptree detectedNode;
|
||
|
|
detectedNode.put("user_id", faces[i].userId);
|
||
|
|
detectedNode.put("user_name", faces[i].userName);
|
||
|
|
detectedNode.put("similarity", faces[i].similarity);
|
||
|
|
detectedNode.put("is_unknown", faces[i].isUnknown);
|
||
|
|
detectedNode.put("prob", faces[i].confidence);
|
||
|
|
detectedNode.put("x", faces[i].box.x);
|
||
|
|
detectedNode.put("y", faces[i].box.y);
|
||
|
|
detectedNode.put("width", faces[i].box.width);
|
||
|
|
detectedNode.put("height", faces[i].box.height);
|
||
|
|
detectedNode.put("extra_info", faces[i].extraInformation);
|
||
|
|
// we might add masks into this using comma seperated string
|
||
|
|
detectedObjects.push_back(std::make_pair("", detectedNode));
|
||
|
|
}
|
||
|
|
root.add_child("results", detectedObjects);
|
||
|
|
std::ostringstream stream;
|
||
|
|
boost::property_tree::write_json(stream, root,false);
|
||
|
|
std::string faceResult = stream.str();
|
||
|
|
return faceResult;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string ANSFRHelper::FaceAttributeToJsonString(Face face) {
|
||
|
|
boost::property_tree::ptree root;
|
||
|
|
boost::property_tree::ptree detectedObjects;
|
||
|
|
boost::property_tree::ptree detectedNode;
|
||
|
|
|
||
|
|
if(face.isAntispoofingEnabled()) {
|
||
|
|
if (face.isReal()) {
|
||
|
|
detectedNode.put("Antispoofing", "Real");
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
detectedNode.put("Antispoofing", "Fake");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (face.isFaceLivenessEnabled()) {
|
||
|
|
if (face.getFaceLiveness() == 1) {
|
||
|
|
detectedNode.put("FaceLiveness", "Real");
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
detectedNode.put("FaceLiveness", "Fake");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (face.isAgeGenderEnabled()) {
|
||
|
|
if (face.isMale()) {
|
||
|
|
detectedNode.put("Gender", "Male");
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
detectedNode.put("Gender", "Female");
|
||
|
|
}
|
||
|
|
int age = face.getAge();
|
||
|
|
if (age < 12)
|
||
|
|
detectedNode.put("Age", "Kids");
|
||
|
|
else if (age < 20)
|
||
|
|
detectedNode.put("Age", "Young Adults");
|
||
|
|
else if (age < 60)
|
||
|
|
detectedNode.put("Age", "Adults");
|
||
|
|
else
|
||
|
|
detectedNode.put("Age", "Elders");
|
||
|
|
|
||
|
|
}
|
||
|
|
if (face.isEmotionsEnabled()) {
|
||
|
|
const auto& [emotionName, emotionConfidence] = face.getMainEmotion();
|
||
|
|
std::ostringstream emotionStr;
|
||
|
|
std::string adjustedEmotionName = emotionName;
|
||
|
|
if (emotionName == "Sad" || emotionName == "sad")
|
||
|
|
adjustedEmotionName = "Attentive";
|
||
|
|
else if (emotionName == "Happy" || emotionName == "happy")
|
||
|
|
adjustedEmotionName = "Happy";
|
||
|
|
else if (emotionName == "Neutral" || emotionName == "neutral")
|
||
|
|
adjustedEmotionName = "Neutral";
|
||
|
|
else if (emotionName == "Angry" || emotionName == "angry")
|
||
|
|
adjustedEmotionName = "Attentive";
|
||
|
|
else if (emotionName == "Surprised" || emotionName == "surprised")
|
||
|
|
adjustedEmotionName = "Attentive";
|
||
|
|
else if (emotionName == "Disgusted" || emotionName == "disgusted")
|
||
|
|
adjustedEmotionName = "Attentive";
|
||
|
|
else if (emotionName == "Fearful" || emotionName == "fearful")
|
||
|
|
adjustedEmotionName = "Attentive";
|
||
|
|
else if (emotionName == "Anger" || emotionName == "anger")
|
||
|
|
adjustedEmotionName = "Attentive";
|
||
|
|
else
|
||
|
|
adjustedEmotionName = "Neutral";
|
||
|
|
|
||
|
|
emotionStr << adjustedEmotionName << " (" << static_cast<int>(emotionConfidence * 100) << "%)";
|
||
|
|
detectedNode.put("Emotion", emotionStr.str());
|
||
|
|
}
|
||
|
|
|
||
|
|
if (face.isHeadPoseEnabled()) {
|
||
|
|
const auto& headPose = face.getHeadPose();
|
||
|
|
std::ostringstream headPoseStr;
|
||
|
|
headPoseStr << "(" << static_cast<int>(headPose.angle_r) << ", "
|
||
|
|
<< static_cast<int>(headPose.angle_p) << ", "
|
||
|
|
<< static_cast<int>(headPose.angle_y) << ")";
|
||
|
|
detectedNode.put("Pose(r,p,y)", headPoseStr.str());
|
||
|
|
}
|
||
|
|
|
||
|
|
// we might add masks into this using comma seperated string
|
||
|
|
detectedObjects.push_back(std::make_pair("", detectedNode));
|
||
|
|
root.add_child("attributes", detectedObjects);
|
||
|
|
std::ostringstream stream;
|
||
|
|
boost::property_tree::write_json(stream, root,false);
|
||
|
|
std::string faceResult = stream.str();
|
||
|
|
return faceResult;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string ANSFRHelper::UserRecordToJsonString(const UserRecord userRecord) {
|
||
|
|
boost::property_tree::ptree root;
|
||
|
|
boost::property_tree::ptree faceIds;
|
||
|
|
for (int i = 0; i < userRecord.FaceIds.size(); i++) {
|
||
|
|
boost::property_tree::ptree faceIdNode;
|
||
|
|
faceIdNode.put("", userRecord.FaceIds[i]);
|
||
|
|
faceIds.push_back(std::make_pair("", faceIdNode));
|
||
|
|
}
|
||
|
|
root.put("user_id", userRecord.UserId);
|
||
|
|
root.put("user_code", userRecord.UserCode);
|
||
|
|
root.put("user_username", userRecord.UserName);
|
||
|
|
root.add_child("face_ids", faceIds);
|
||
|
|
std::ostringstream stream;
|
||
|
|
boost::property_tree::write_json(stream, root,false);
|
||
|
|
std::string userRecordResult = stream.str();
|
||
|
|
return userRecordResult;
|
||
|
|
}
|
||
|
|
std::string ANSFRHelper::UserRecordsToJsonString(const std::vector<UserRecord> userRecords) {
|
||
|
|
boost::property_tree::ptree root;
|
||
|
|
boost::property_tree::ptree userRecordNodes;
|
||
|
|
for (int i=0; i < userRecords.size(); i++) {
|
||
|
|
boost::property_tree::ptree userRecordNode;
|
||
|
|
boost::property_tree::ptree faceIds;
|
||
|
|
UserRecord userRecord = userRecords[i];
|
||
|
|
for (int j = 0; j < userRecord.FaceIds.size(); j++) {
|
||
|
|
boost::property_tree::ptree faceIdNode;
|
||
|
|
faceIdNode.put("", userRecord.FaceIds[j]);
|
||
|
|
faceIds.push_back(std::make_pair("", faceIdNode));
|
||
|
|
}
|
||
|
|
userRecordNode.put("user_id", userRecord.UserId);
|
||
|
|
userRecordNode.put("user_code", userRecord.UserCode);
|
||
|
|
userRecordNode.put("user_username", userRecord.UserName);
|
||
|
|
userRecordNode.add_child("face_ids", faceIds);
|
||
|
|
userRecordNodes.push_back(std::make_pair("", userRecordNode));
|
||
|
|
}
|
||
|
|
root.add_child("user_records", userRecordNodes);
|
||
|
|
std::ostringstream stream;
|
||
|
|
boost::property_tree::write_json(stream, root,false);
|
||
|
|
std::string userRecordResult = stream.str();
|
||
|
|
return userRecordResult;
|
||
|
|
}
|
||
|
|
std::string ANSFRHelper::FaceRecordToJsonString(const FaceRecord faceRecord) {
|
||
|
|
boost::property_tree::ptree root;
|
||
|
|
root.put("face_id", faceRecord.FaceId);
|
||
|
|
root.put("user_id", faceRecord.UserId);
|
||
|
|
root.put("image_path", faceRecord.ImagePath);
|
||
|
|
std::ostringstream stream;
|
||
|
|
boost::property_tree::write_json(stream, root,false);
|
||
|
|
std::string userRecordResult = stream.str();
|
||
|
|
return userRecordResult;
|
||
|
|
}
|
||
|
|
std::string ANSFRHelper::FaceRecordsToJsonString(const std::vector<FaceRecord> faceRecords) {
|
||
|
|
boost::property_tree::ptree root;
|
||
|
|
boost::property_tree::ptree faceNodes;
|
||
|
|
for (int i = 0; i < faceRecords.size(); i++) {
|
||
|
|
FaceRecord faceRecord = faceRecords[i];
|
||
|
|
boost::property_tree::ptree faceNode;
|
||
|
|
faceNode.put("face_id", faceRecord.FaceId);
|
||
|
|
faceNode.put("user_id", faceRecord.UserId);
|
||
|
|
faceNode.put("image_path", faceRecord.ImagePath);
|
||
|
|
faceNodes.push_back(std::make_pair("", faceNode));
|
||
|
|
}
|
||
|
|
root.add_child("face_records", faceNodes);
|
||
|
|
std::ostringstream stream;
|
||
|
|
boost::property_tree::write_json(stream, root,false);
|
||
|
|
std::string userRecordResult = stream.str();
|
||
|
|
return userRecordResult;
|
||
|
|
}
|
||
|
|
|
||
|
|
cv::Mat ANSFRHelper::PreprocessImg(cv::Mat& img, int input_w, int input_h) {
|
||
|
|
try {
|
||
|
|
int w, h, x, y;
|
||
|
|
float r_w = input_w / (img.cols * 1.0);
|
||
|
|
float r_h = input_h / (img.rows * 1.0);
|
||
|
|
if (r_h > r_w) {
|
||
|
|
w = input_w;
|
||
|
|
h = r_w * img.rows;
|
||
|
|
x = 0;
|
||
|
|
y = (input_h - h) / 2;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
w = r_h * img.cols;
|
||
|
|
h = input_h;
|
||
|
|
x = (input_w - w) / 2;
|
||
|
|
y = 0;
|
||
|
|
}
|
||
|
|
cv::Mat re(h, w, CV_8UC3);
|
||
|
|
cv::resize(img, re, re.size(), 0, 0, cv::INTER_LINEAR);
|
||
|
|
cv::Mat out(input_h, input_w, CV_8UC3, cv::Scalar(128, 128, 128));
|
||
|
|
re.copyTo(out(cv::Rect(x, y, re.cols, re.rows)));
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
catch (std::exception& e) {
|
||
|
|
cv::Mat result;
|
||
|
|
std::cerr << "ANSFRHelper::PreprocessImg" << e.what();
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int ANSFRHelper::ReadFilesInDir(const char* p_dir_name, std::vector<std::string>& file_names) {
|
||
|
|
try {
|
||
|
|
DIR* p_dir = opendir(p_dir_name);
|
||
|
|
if (p_dir == nullptr) {
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
struct dirent* p_file = nullptr;
|
||
|
|
while ((p_file = readdir(p_dir)) != nullptr) {
|
||
|
|
if (strcmp(p_file->d_name, ".") != 0 &&
|
||
|
|
strcmp(p_file->d_name, "..") != 0) {
|
||
|
|
std::string cur_file_name(p_file->d_name);
|
||
|
|
file_names.push_back(cur_file_name);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
closedir(p_dir);
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
catch (std::exception& e) {
|
||
|
|
std::cerr << "ANSFRHelper::ReadFilesInDir" << e.what();
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
cv::Rect ANSFRHelper::GetRectAdaptLandmark(cv::Mat& img, int input_w, int input_h, float bbox[4], float lmk[10]) {
|
||
|
|
int l, r, t, b;
|
||
|
|
float r_w = input_w / (img.cols * 1.0);
|
||
|
|
float r_h = input_h / (img.rows * 1.0);
|
||
|
|
if (r_h > r_w) {
|
||
|
|
l = bbox[0] / r_w;
|
||
|
|
r = bbox[2] / r_w;
|
||
|
|
t = (bbox[1] - (input_h - r_w * img.rows) / 2) / r_w;
|
||
|
|
b = (bbox[3] - (input_h - r_w * img.rows) / 2) / r_w;
|
||
|
|
for (int i = 0; i < 10; i += 2) {
|
||
|
|
lmk[i] /= r_w;
|
||
|
|
lmk[i + 1] = (lmk[i + 1] - (input_h - r_w * img.rows) / 2) / r_w;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
l = (bbox[0] - (input_w - r_h * img.cols) / 2) / r_h;
|
||
|
|
r = (bbox[2] - (input_w - r_h * img.cols) / 2) / r_h;
|
||
|
|
t = bbox[1] / r_h;
|
||
|
|
b = bbox[3] / r_h;
|
||
|
|
for (int i = 0; i < 10; i += 2) {
|
||
|
|
lmk[i] = (lmk[i] - (input_w - r_h * img.cols) / 2) / r_h;
|
||
|
|
lmk[i + 1] /= r_h;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return cv::Rect(l, t, r - l, b - t);
|
||
|
|
}
|
||
|
|
|
||
|
|
float ANSFRHelper::IOU(float lbox[4], float rbox[4]) {
|
||
|
|
float interBox[] = {
|
||
|
|
MAX(lbox[0], rbox[0]), //left
|
||
|
|
MIN(lbox[2], rbox[2]), //right
|
||
|
|
MAX(lbox[1], rbox[1]), //top
|
||
|
|
MIN(lbox[3], rbox[3]), //bottom
|
||
|
|
};
|
||
|
|
|
||
|
|
if (interBox[2] > interBox[3] || interBox[0] > interBox[1])
|
||
|
|
return 0.0f;
|
||
|
|
|
||
|
|
float interBoxS = (interBox[1] - interBox[0]) * (interBox[3] - interBox[2]);
|
||
|
|
return interBoxS / ((lbox[2] - lbox[0]) * (lbox[3] - lbox[1]) + (rbox[2] - rbox[0]) * (rbox[3] - rbox[1]) - interBoxS + 0.000001f);
|
||
|
|
}
|
||
|
|
|
||
|
|
bool ANSFRHelper::CMP(const Detection& a, const Detection& b) {
|
||
|
|
return a.class_confidence > b.class_confidence;
|
||
|
|
}
|
||
|
|
|
||
|
|
void ANSFRHelper::NMS(std::vector<Detection>& res, float* output, float nms_thresh) {
|
||
|
|
std::vector<Detection> dets;
|
||
|
|
for (int i = 0; i < output[0]; i++) {
|
||
|
|
if (output[15 * i + 1 + 4] <= 0.1) continue;
|
||
|
|
Detection det;
|
||
|
|
memcpy(&det, &output[15 * i + 1], sizeof(Detection));
|
||
|
|
dets.push_back(det);
|
||
|
|
}
|
||
|
|
std::sort(dets.begin(), dets.end(), ANSFRHelper::CMP);
|
||
|
|
for (size_t m = 0; m < dets.size(); ++m) {
|
||
|
|
auto& item = dets[m];
|
||
|
|
res.push_back(item);
|
||
|
|
//std::cout << item.class_confidence << " bbox " << item.bbox[0] << ", " << item.bbox[1] << ", " << item.bbox[2] << ", " << item.bbox[3] << std::endl;
|
||
|
|
for (size_t n = m + 1; n < dets.size(); ++n) {
|
||
|
|
if (ANSFRHelper::IOU(item.bbox, dets[n].bbox) > nms_thresh) {
|
||
|
|
dets.erase(dets.begin() + n);
|
||
|
|
--n;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
std::map<std::string, Weights> ANSFRHelper::LoadWeights(const std::string file) {
|
||
|
|
std::cout << "Loading weights: " << file << std::endl;
|
||
|
|
std::map<std::string, Weights> weightMap;
|
||
|
|
|
||
|
|
// Open weights file
|
||
|
|
std::ifstream input(file);
|
||
|
|
assert(input.is_open() && "Unable to load weight file.");
|
||
|
|
|
||
|
|
// Read number of weight blobs
|
||
|
|
int32_t count;
|
||
|
|
input >> count;
|
||
|
|
assert(count > 0 && "Invalid weight map file.");
|
||
|
|
|
||
|
|
while (count--)
|
||
|
|
{
|
||
|
|
Weights wt{ DataType::kFLOAT, nullptr, 0 };
|
||
|
|
uint32_t size;
|
||
|
|
|
||
|
|
// Read name and type of blob
|
||
|
|
std::string name;
|
||
|
|
input >> name >> std::dec >> size;
|
||
|
|
wt.type = DataType::kFLOAT;
|
||
|
|
|
||
|
|
// Load blob
|
||
|
|
uint32_t* val = reinterpret_cast<uint32_t*>(malloc(sizeof(val) * size));
|
||
|
|
for (uint32_t x = 0, y = size; x < y; ++x)
|
||
|
|
{
|
||
|
|
input >> std::hex >> val[x];
|
||
|
|
}
|
||
|
|
wt.values = val;
|
||
|
|
|
||
|
|
wt.count = size;
|
||
|
|
weightMap[name] = wt;
|
||
|
|
}
|
||
|
|
|
||
|
|
return weightMap;
|
||
|
|
}
|
||
|
|
Weights ANSFRHelper::GetWeights(std::map<std::string, Weights>& weightMap, std::string key) {
|
||
|
|
if (weightMap.count(key) != 1) {
|
||
|
|
std::cerr << key << " not existed in weight map, fatal error!!!" << std::endl;
|
||
|
|
exit(-1);
|
||
|
|
}
|
||
|
|
return weightMap[key];
|
||
|
|
}
|
||
|
|
|
||
|
|
IScaleLayer* ANSFRHelper::AddBatchNorm2D(INetworkDefinition* network, std::map<std::string, Weights>& weightMap, ITensor& input, std::string lname, float eps) {
|
||
|
|
float* gamma = (float*)weightMap[lname + ".weight"].values;
|
||
|
|
float* beta = (float*)weightMap[lname + ".bias"].values;
|
||
|
|
float* mean = (float*)weightMap[lname + ".running_mean"].values;
|
||
|
|
float* var = (float*)weightMap[lname + ".running_var"].values;
|
||
|
|
int len = weightMap[lname + ".running_var"].count;
|
||
|
|
|
||
|
|
float* scval = reinterpret_cast<float*>(malloc(sizeof(float) * len));
|
||
|
|
for (int i = 0; i < len; i++) {
|
||
|
|
scval[i] = gamma[i] / sqrt(var[i] + eps);
|
||
|
|
}
|
||
|
|
Weights scale{ DataType::kFLOAT, scval, len };
|
||
|
|
|
||
|
|
float* shval = reinterpret_cast<float*>(malloc(sizeof(float) * len));
|
||
|
|
for (int i = 0; i < len; i++) {
|
||
|
|
shval[i] = beta[i] - mean[i] * gamma[i] / sqrt(var[i] + eps);
|
||
|
|
}
|
||
|
|
Weights shift{ DataType::kFLOAT, shval, len };
|
||
|
|
|
||
|
|
float* pval = reinterpret_cast<float*>(malloc(sizeof(float) * len));
|
||
|
|
for (int i = 0; i < len; i++) {
|
||
|
|
pval[i] = 1.0;
|
||
|
|
}
|
||
|
|
Weights power{ DataType::kFLOAT, pval, len };
|
||
|
|
|
||
|
|
weightMap[lname + ".scale"] = scale;
|
||
|
|
weightMap[lname + ".shift"] = shift;
|
||
|
|
weightMap[lname + ".power"] = power;
|
||
|
|
IScaleLayer* scale_1 = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power);
|
||
|
|
assert(scale_1);
|
||
|
|
return scale_1;
|
||
|
|
}
|
||
|
|
/// <summary>
|
||
|
|
/// MatMul
|
||
|
|
/// </summary>
|
||
|
|
//MatMul::MatMul() {
|
||
|
|
// try {
|
||
|
|
// ANSFRHelper::ANSFRHelper::CheckCudaStatus(cudaMalloc(&workspace, workspaceSize));
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtCreate(<Handle));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaStreamCreate(&stream));
|
||
|
|
// }
|
||
|
|
// catch (std::exception& e) {
|
||
|
|
// std::cerr << "MatMul::MatMul" << e.what();
|
||
|
|
// }
|
||
|
|
//}
|
||
|
|
//void MatMul::Init(float* knownEmbeds, int numRow, int numCol) {
|
||
|
|
// try {
|
||
|
|
// m = static_cast<const int>(numRow);
|
||
|
|
// k = static_cast<const int>(numCol);
|
||
|
|
// lda = static_cast<const int>(numCol);
|
||
|
|
// ldb = static_cast<const int>(numCol);
|
||
|
|
// ldc = static_cast<const int>(numRow);
|
||
|
|
|
||
|
|
// // alloc and copy known embeddings to GPU
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast<void**>(&dA), m * k * sizeof(float)));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(dA, knownEmbeds, m * k * sizeof(float), cudaMemcpyHostToDevice, stream));
|
||
|
|
|
||
|
|
// // create operation desciriptor; see cublasLtMatmulDescAttributes_t for details about defaults;
|
||
|
|
// // here we just need to set the transforms for A and B
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescCreate(&operationDesc, computeType, cudaDataType));
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)));
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transb)));
|
||
|
|
|
||
|
|
// // create matrix descriptors, we are good with the details here so no need to set any extra attributes
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Adesc, cudaDataType, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda));
|
||
|
|
|
||
|
|
// // create preference handle; here we could use extra attributes to disable tensor ops or to make sure algo selected
|
||
|
|
// // will work with badly aligned A, B, C; here for simplicity we just assume A,B,C are always well aligned (e.g.
|
||
|
|
// // directly come from cudaMalloc)
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceCreate(&preference));
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceSetAttribute(preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)));
|
||
|
|
// }
|
||
|
|
// catch (std::exception& e) {
|
||
|
|
// std::cerr << "MatMul::Init" << e.what();
|
||
|
|
// }
|
||
|
|
//}
|
||
|
|
//void MatMul::Calculate(float* embeds, int embedCount, float* outputs) {
|
||
|
|
// try {
|
||
|
|
// n = embedCount;
|
||
|
|
|
||
|
|
// // Allocate arrays on GPU
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast<void**>(&dB), k * n * sizeof(float)));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast<void**>(&dC), m * n * sizeof(float)));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(dB, embeds, k * n * sizeof(float), cudaMemcpyHostToDevice, stream));
|
||
|
|
|
||
|
|
// // create matrix descriptors, we are good with the details here so no need to set any extra attributes
|
||
|
|
// cublasLtMatrixLayout_t Bdesc = NULL, Cdesc = NULL;
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Bdesc, cudaDataType, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb));
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Cdesc, cudaDataType, m, n, ldc));
|
||
|
|
|
||
|
|
// // we just need the best available heuristic to try and run matmul. There is no guarantee this will work, e.g. if A
|
||
|
|
// // is badly aligned, you can request more (e.g. 32) algos and try to run them one by one until something works
|
||
|
|
// int returnedResults = 0;
|
||
|
|
// cublasLtMatmulHeuristicResult_t heuristicResult = {};
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulAlgoGetHeuristic(ltHandle, operationDesc, Adesc, Bdesc, Cdesc, Cdesc, preference, 1, &heuristicResult, &returnedResults));
|
||
|
|
// if (returnedResults == 0) {
|
||
|
|
// ANSFRHelper::CheckCublasStatus(CUBLAS_STATUS_NOT_SUPPORTED);
|
||
|
|
// }
|
||
|
|
|
||
|
|
// // Do the actual multiplication
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmul(ltHandle, operationDesc, &alpha, dA, Adesc, dB, Bdesc, &beta, dC, Cdesc, dC, Cdesc, &heuristicResult.algo, workspace,
|
||
|
|
// workspaceSize, stream));
|
||
|
|
|
||
|
|
// // Cleanup: descriptors are no longer needed as all GPU work was already enqueued
|
||
|
|
// if (Cdesc)
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Cdesc));
|
||
|
|
// if (Bdesc)
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Bdesc));
|
||
|
|
|
||
|
|
// // Copy the result on host memory
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(outputs, dC, m * n * sizeof(float), cudaMemcpyDeviceToHost, stream));
|
||
|
|
|
||
|
|
// // CUDA stream sync
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaStreamSynchronize(stream));
|
||
|
|
|
||
|
|
// // Free GPU memory
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaFree(dB));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaFree(dC));
|
||
|
|
// }
|
||
|
|
// catch (std::exception& e) {
|
||
|
|
// std::cerr <<"MatMul::Calculate"<< e.what();
|
||
|
|
// }
|
||
|
|
//}
|
||
|
|
//MatMul::~MatMul() {
|
||
|
|
// try {
|
||
|
|
// if (preference)
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceDestroy(preference));
|
||
|
|
// if (Adesc)
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Adesc));
|
||
|
|
// if (operationDesc)
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescDestroy(operationDesc));
|
||
|
|
|
||
|
|
// ANSFRHelper::CheckCublasStatus(cublasLtDestroy(ltHandle));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaFree(dA));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaFree(workspace));
|
||
|
|
// ANSFRHelper::CheckCudaStatus(cudaStreamDestroy(stream));
|
||
|
|
// }
|
||
|
|
|
||
|
|
// catch (std::exception& e) {
|
||
|
|
// std::cout << "MatMul::~MatMul" << e.what();
|
||
|
|
// }
|
||
|
|
|
||
|
|
//}
|
||
|
|
//Int8EntropyCalibrator2::Int8EntropyCalibrator2(int batchsize, int input_w, int input_h, const char* img_dir, const char* calib_table_name, const char* input_blob_name, bool read_cache)
|
||
|
|
// : batchsize_(batchsize)
|
||
|
|
// , input_w_(input_w)
|
||
|
|
// , input_h_(input_h)
|
||
|
|
// , img_idx_(0)
|
||
|
|
// , img_dir_(img_dir)
|
||
|
|
// , calib_table_name_(calib_table_name)
|
||
|
|
// , input_blob_name_(input_blob_name)
|
||
|
|
// , read_cache_(read_cache)
|
||
|
|
//{
|
||
|
|
// input_count_ = 3 * input_w * input_h * batchsize;
|
||
|
|
// CUDACHECK(cudaMalloc(&device_input_, input_count_ * sizeof(float)));
|
||
|
|
// ANSFRHelper::ReadFilesInDir(img_dir, img_files_);
|
||
|
|
//}
|
||
|
|
//Int8EntropyCalibrator2::~Int8EntropyCalibrator2()
|
||
|
|
//{
|
||
|
|
// CUDACHECK(cudaFree(device_input_));
|
||
|
|
//}
|
||
|
|
//int Int8EntropyCalibrator2::getBatchSize() const TRT_NOEXCEPT
|
||
|
|
//{
|
||
|
|
// return batchsize_;
|
||
|
|
//}
|
||
|
|
//bool Int8EntropyCalibrator2::getBatch(void* bindings[], const char* names[], int nbBindings) TRT_NOEXCEPT
|
||
|
|
//{
|
||
|
|
// if (img_idx_ + batchsize_ > (int)img_files_.size()) {
|
||
|
|
// return false;
|
||
|
|
// }
|
||
|
|
|
||
|
|
// std::vector<cv::Mat> input_imgs_;
|
||
|
|
// for (int i = img_idx_; i < img_idx_ + batchsize_; i++) {
|
||
|
|
// std::cout << img_files_[i] << " " << i << std::endl;
|
||
|
|
// cv::Mat temp = cv::imread(img_dir_ + img_files_[i]);
|
||
|
|
// if (temp.empty()) {
|
||
|
|
// std::cerr << "Fatal error: image cannot open!" << std::endl;
|
||
|
|
// return false;
|
||
|
|
// }
|
||
|
|
// cv::Mat pr_img = ANSFRHelper::PreprocessImg(temp, input_w_, input_h_);
|
||
|
|
// input_imgs_.push_back(pr_img);
|
||
|
|
// }
|
||
|
|
// img_idx_ += batchsize_;
|
||
|
|
// cv::Mat blob = cv::dnn::blobFromImages(input_imgs_, 1.0, cv::Size(input_w_, input_h_), cv::Scalar(104, 117, 123), false, false);
|
||
|
|
|
||
|
|
// CUDACHECK(cudaMemcpy(device_input_, blob.ptr<float>(0), input_count_ * sizeof(float), cudaMemcpyHostToDevice));
|
||
|
|
// assert(!strcmp(names[0], input_blob_name_));
|
||
|
|
// bindings[0] = device_input_;
|
||
|
|
// return true;
|
||
|
|
//}
|
||
|
|
//const void* Int8EntropyCalibrator2::readCalibrationCache(size_t& length) TRT_NOEXCEPT
|
||
|
|
//{
|
||
|
|
// std::cout << "reading calib cache: " << calib_table_name_ << std::endl;
|
||
|
|
// calib_cache_.clear();
|
||
|
|
// std::ifstream input(calib_table_name_, std::ios::binary);
|
||
|
|
// input >> std::noskipws;
|
||
|
|
// if (read_cache_ && input.good())
|
||
|
|
// {
|
||
|
|
// std::copy(std::istream_iterator<char>(input), std::istream_iterator<char>(), std::back_inserter(calib_cache_));
|
||
|
|
// }
|
||
|
|
// length = calib_cache_.size();
|
||
|
|
// return length ? calib_cache_.data() : nullptr;
|
||
|
|
//}
|
||
|
|
//void Int8EntropyCalibrator2::writeCalibrationCache(const void* cache, size_t length) TRT_NOEXCEPT
|
||
|
|
//{
|
||
|
|
// std::cout << "writing calib cache: " << calib_table_name_ << " size: " << length << std::endl;
|
||
|
|
// std::ofstream output(calib_table_name_, std::ios::binary);
|
||
|
|
// output.write(reinterpret_cast<const char*>(cache), length);
|
||
|
|
//}
|
||
|
|
}
|