#include "ANSFRCommon.h" #include "dirent.h" #include #include #include #include #include namespace ANSCENTER { constexpr size_t ndetections = 200; template T GetData(const boost::property_tree::ptree& pt, const std::string& key) { T ret; if (boost::optional data = pt.get_optional(key)) { ret = data.get(); } return ret; } Face::Face(size_t id, cv::Rect& location) : _location(location), _intensity_mean(0.f), _id(id), _age(-1), _maleScore(0), _femaleScore(0), _realFaceConfidence(0), _isAgeGenderEnabled(false), _isEmotionsEnabled(false), _isHeadPoseEnabled(false), _isLandmarksEnabled(false), _isAntispoofingEnabled(false) { HeadPoseResults headPose; headPose.angle_p = 0.f; headPose.angle_r = 0.f; headPose.angle_y = 0.f; _headPose = headPose; } void Face::updateAge(float value) { _age = (_age == -1) ? value : 0.95f * _age + 0.05f * value; } void Face::updateGender(float value) { if (value < 0) return; if (value > 0.5) { _maleScore += value - 0.5f; } else { _femaleScore += 0.5f - value; } } void Face::updateEmotions(std::map values) { for (auto& kv : values) { if (_emotions.find(kv.first) == _emotions.end()) { _emotions[kv.first] = kv.second; } else { _emotions[kv.first] = 0.9f * _emotions[kv.first] + 0.1f * kv.second; } } } void Face::updateHeadPose(HeadPoseResults values) { _headPose = values; } void Face::updateLandmarks(std::vector values) { _landmarks = std::move(values); } void Face::updateRealFaceConfidence(float value) { _realFaceConfidence = value; } void Face::updateFaceLiveness(int value) { _faceLiveness = value; } int Face::getAge() { return static_cast(std::floor(_age + 0.5f)); } bool Face::isMale() { return _maleScore > _femaleScore; } bool Face::isReal() { return _realFaceConfidence > 0.4; } float Face::getAntispoofingScore() { return _realFaceConfidence; } int Face::getFaceLiveness() { return _faceLiveness; } std::map Face::getEmotions() { return _emotions; } std::pair Face::getMainEmotion() { auto x = std::max_element(_emotions.begin(), _emotions.end(), [](const std::pair& p1, const std::pair& p2) { return p1.second < p2.second; }); return std::make_pair(x->first, x->second); } HeadPoseResults Face::getHeadPose() { return _headPose; } const std::vector& Face::getLandmarks() { return _landmarks; } size_t Face::getId() { return _id; } void Face::ageGenderEnable(bool value) { _isAgeGenderEnabled = value; } void Face::emotionsEnable(bool value) { _isEmotionsEnabled = value; } void Face::headPoseEnable(bool value) { _isHeadPoseEnabled = value; } void Face::landmarksEnable(bool value) { _isLandmarksEnabled = value; } void Face::antispoofingEnable(bool value) { _isAntispoofingEnabled = value; } void Face::faceLivenessEnable(bool value) { _isFaceLivenessEnabled = value; } bool Face::isAgeGenderEnabled() { return _isAgeGenderEnabled; } bool Face::isEmotionsEnabled() { return _isEmotionsEnabled; } bool Face::isHeadPoseEnabled() { return _isHeadPoseEnabled; } bool Face::isLandmarksEnabled() { return _isLandmarksEnabled; } bool Face::isAntispoofingEnabled() { return _isAntispoofingEnabled; } bool Face::isFaceLivenessEnabled() { return _isFaceLivenessEnabled; } float calcIoU(cv::Rect& src, cv::Rect& dst) { cv::Rect i = src & dst; cv::Rect u = src | dst; return static_cast(i.area()) / static_cast(u.area()); } float calcMean(const cv::Mat& src) { cv::Mat tmp; cv::cvtColor(src, tmp, cv::COLOR_BGR2GRAY); cv::Scalar mean = cv::mean(tmp); return static_cast(mean[0]); } Face matchFace(cv::Rect rect, std::list& faces) { Face face(0,rect); float maxIoU = 0.55f; for (auto&& f : faces) { float iou = calcIoU(rect, f._location); if (iou > maxIoU) { face = f; maxIoU = iou; } } return face; } BaseDetection::BaseDetection(const std::string& pathToModel, bool doRawOutputMessages) : pathToModel(pathToModel), doRawOutputMessages(doRawOutputMessages) { } bool BaseDetection::enabled() const { return bool(request); } AntispoofingClassifier::AntispoofingClassifier(const std::string& pathToModel, bool doRawOutputMessages) : BaseDetection(pathToModel, doRawOutputMessages) { _modelFilePath = pathToModel; } float AntispoofingClassifier::runInfer(const cv::Mat& frame) { std::lock_guard lock(_mutex); try { cv::Mat face = frame.clone(); auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }; resize2tensor(face, inSlice); request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }); request.infer(); face.release(); float r = request.get_output_tensor().data()[0] * 100; return r; } catch (const std::exception& error) { std::cerr << "Error: " << error.what() << std::endl; return -1; } } std::shared_ptr AntispoofingClassifier::read(const ov::Core& core) { slog::info << "Reading model: " << _modelFilePath << slog::endl; std::shared_ptr model = core.read_model(_modelFilePath); ov::preprocess::PrePostProcessor ppp(model); ppp.input().tensor(). set_element_type(ov::element::u8). set_layout("NHWC"); ppp.input().preprocess().convert_layout("NCHW"); ppp.output().tensor().set_element_type(ov::element::f32); model = ppp.build(); inShape = model->input().get_shape(); inShape[0] = ndetections; //ov::set_batch(model, { 1, int64_t(ndetections) }); ov::set_batch(model, { 1 }); // Force single batch for all inputs return model; } AgeGenderDetection::AgeGenderDetection(const std::string& pathToModel, bool doRawOutputMessages) : BaseDetection(pathToModel, doRawOutputMessages) { _modelFilePath = pathToModel; } AgeGenderResults AgeGenderDetection::runInfer(const cv::Mat& frame) { std::lock_guard lock(_mutex); try { cv::Mat face = frame.clone(); auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }; resize2tensor(face, inSlice); request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }); request.infer(); face.release(); AgeGenderResults r = { request.get_tensor(outputAge).data()[0] * 100, request.get_tensor(outputGender).data()[0 * 2 + 1] }; return r; } catch (const std::exception& error) { AgeGenderResults r = { -1, -1 }; return r; } } std::shared_ptr AgeGenderDetection::read(const ov::Core& core) { slog::info << "Reading model: " << _modelFilePath << slog::endl; std::shared_ptr model = core.read_model(_modelFilePath); outputAge = "age_conv3"; outputGender = "prob"; ov::preprocess::PrePostProcessor ppp(model); ppp.input().tensor(). set_element_type(ov::element::u8). set_layout("NHWC"); ppp.input().preprocess(). convert_element_type(ov::element::f32). convert_layout("NCHW"); ppp.output(outputAge).tensor().set_element_type(ov::element::f32); ppp.output(outputGender).tensor().set_element_type(ov::element::f32); model = ppp.build(); inShape = model->input().get_shape(); inShape[0] = ndetections; //ov::set_batch(model, { 1, int64_t(ndetections) }); ov::set_batch(model, { 1 }); // Force single batch for all inputs return model; } HeadPoseDetection::HeadPoseDetection(const std::string& pathToModel, bool doRawOutputMessages) : BaseDetection(pathToModel, doRawOutputMessages), outputAngleR("angle_r_fc"), outputAngleP("angle_p_fc"), outputAngleY("angle_y_fc") { _modelFilePath = pathToModel; } HeadPoseResults HeadPoseDetection::runInfer(const cv::Mat& frame) { std::lock_guard lock(_mutex); try { cv::Mat face = frame.clone(); auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }; resize2tensor(face, inSlice); request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }); request.infer(); face.release(); HeadPoseResults r = { request.get_tensor(outputAngleR).data()[0], request.get_tensor(outputAngleP).data()[0], request.get_tensor(outputAngleY).data()[0] }; return r; } catch (const std::exception& error) { HeadPoseResults r = { -1, -1, -1 }; return r; } } std::shared_ptr HeadPoseDetection::read(const ov::Core& core) { std::shared_ptr model = core.read_model(_modelFilePath); ov::preprocess::PrePostProcessor ppp(model); ppp.input().tensor(). set_element_type(ov::element::u8). set_layout("NHWC"); ppp.input().preprocess().convert_layout("NCHW"); ppp.output(outputAngleR).tensor().set_element_type(ov::element::f32); ppp.output(outputAngleP).tensor().set_element_type(ov::element::f32); ppp.output(outputAngleY).tensor().set_element_type(ov::element::f32); model = ppp.build(); inShape = model->input().get_shape(); inShape[0] = ndetections; //ov::set_batch(model, { 1, int64_t(ndetections) }); ov::set_batch(model, { 1 }); // Force single batch for all inputs return model; } EmotionsDetection::EmotionsDetection(const std::string& pathToModel, bool doRawOutputMessages) : BaseDetection(pathToModel, doRawOutputMessages) { _modelFilePath = pathToModel; } std::map EmotionsDetection::runInfer(const cv::Mat& frame) { std::lock_guard lock(_mutex); try { cv::Mat face = frame.clone(); auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }; resize2tensor(face, inSlice); request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }); request.infer(); face.release(); auto tensor = request.get_output_tensor(); auto emotionsVecSize = emotionsVec.size(); size_t numOfChannels = tensor.get_shape().at(1); if (numOfChannels == emotionsVecSize) { const float* emotionsValues = tensor.data(); auto outputIdxPos = emotionsValues + 0 * emotionsVecSize; std::map emotions; for (size_t i = 0; i < emotionsVecSize; i++) { emotions[emotionsVec[i]] = outputIdxPos[i]; } return emotions; } else return { }; } catch (const std::exception& error) { std::cerr << "Error: " << error.what() << std::endl; return { }; } } std::shared_ptr EmotionsDetection::read(const ov::Core& core) { slog::info << "Reading model: " << _modelFilePath << slog::endl; std::shared_ptr model = core.read_model(_modelFilePath); ov::preprocess::PrePostProcessor ppp(model); ppp.input().tensor(). set_element_type(ov::element::u8). set_layout("NHWC"); ppp.input().preprocess().convert_layout("NCHW"); ppp.output().tensor().set_element_type(ov::element::f32); model = ppp.build(); inShape = model->input().get_shape(); inShape[0] = ndetections; //ov::set_batch(model, { 1, int64_t(ndetections) }); ov::set_batch(model, { 1 }); // Force single batch for all inputs return model; } FacialLandmarksDetection::FacialLandmarksDetection(const std::string& pathToModel, bool doRawOutputMessages) : BaseDetection(pathToModel, doRawOutputMessages) { _modelFilePath = pathToModel; } std::vector FacialLandmarksDetection::runInfer(const cv::Mat& frame) { std::lock_guard lock(_mutex); std::vector normedLandmarks; try { cv::Mat face = frame.clone(); auto inSlice = ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }; resize2tensor(face, inSlice); request.set_input_tensor(ov::Tensor{ inTensor, {0, 0, 0, 0}, {1, inShape[1], inShape[2], inShape[3]} }); request.infer(); face.release(); auto tensor = request.get_output_tensor(); size_t n_lm = tensor.get_shape().at(1); const float* normed_coordinates = tensor.data(); auto begin = 0; auto end = begin + n_lm / 2; for (auto i_lm = begin; i_lm < end; ++i_lm) { float normed_x = normed_coordinates[2 * i_lm]; float normed_y = normed_coordinates[2 * i_lm + 1]; normedLandmarks.push_back(normed_x); normedLandmarks.push_back(normed_y); } return normedLandmarks; } catch (const std::exception& error) { std::cerr << "Error: " << error.what() << std::endl; return normedLandmarks; } } std::shared_ptr FacialLandmarksDetection::read(const ov::Core& core) { std::shared_ptr model = core.read_model(_modelFilePath); ov::Shape outShape = model->output().get_shape(); if (outShape.size() != 2 && outShape.back() != 70) { return model; } ov::preprocess::PrePostProcessor ppp(model); ppp.input().tensor(). set_element_type(ov::element::u8). set_layout("NHWC"); ppp.input().preprocess().convert_layout("NCHW"); ppp.output().tensor().set_element_type(ov::element::f32); model = ppp.build(); inShape = model->input().get_shape(); inShape[0] = ndetections; //ov::set_batch(model, { 1, int64_t(ndetections) }); ov::set_batch(model, { 1 }); // Force single batch for all inputs return model; } //void ANSFRHelper::ANSFRHelper::CheckCudaStatus(cudaError_t status) { // try { // if (status != cudaSuccess) { // std::cerr << "CUDA API failed with status " << status << ": " << cudaGetErrorString(status) << std::endl; // } // } // catch (std::exception& e) { // std::cerr << "ANSFRHelper::CheckCublasStatus" << e.what(); // } //} //void ANSFRHelper::ANSFRHelper::CheckCublasStatus(cublasStatus_t status) { // try { // if (status != CUBLAS_STATUS_SUCCESS) { // std::cerr << "cuBLAS API failed with status " << status << std::endl; // } // } // catch (std::exception& e) { // std::cerr << "ANSFRHelper::CheckCublasStatus" << e.what(); // } //} void ANSFRHelper::GetCroppedFaces(const cv::Mat &input, std::vector& outputBbox, int resize_w, int resize_h, std::vector& croppedFaces) { croppedFaces.clear(); cv::Mat frame = input.clone(); if (outputBbox.size() > 0) { for (std::vector::iterator it = outputBbox.begin(); it != outputBbox.end(); it++) { int x1, y1, x2, y2; x1 = (*it).box.x; y1 = (*it).box.y; x2 = (*it).box.x + (*it).box.width; y2 = (*it).box.y + (*it).box.height; cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2)); cv::Mat tempCrop = frame(facePos); struct CroppedFace currFace; cv::resize(tempCrop, currFace.faceMat, cv::Size(resize_h, resize_w), 0, 0, cv::INTER_CUBIC); currFace.face = currFace.faceMat.clone(); currFace.x1 = x1; currFace.y1 = y1; currFace.x2 = x2; currFace.y2 = y2; croppedFaces.push_back(currFace); } } else { // this will be the same image int x1, y1, x2, y2; x1 = 0; y1 = 0; x2 = frame.cols; y2 = frame.rows; cv::Rect facePos(cv::Point(x1, y1), cv::Point(x2, y2)); cv::Mat tempCrop = frame(facePos); struct CroppedFace currFace; cv::resize(tempCrop, currFace.faceMat, cv::Size(resize_h, resize_w), 0, 0, cv::INTER_CUBIC); currFace.face = currFace.faceMat.clone(); currFace.x1 = x1; currFace.y1 = y1; currFace.x2 = x2; currFace.y2 = y2; croppedFaces.push_back(currFace); } frame.release(); } bool ANSFRHelper::LoadConfigFile(std::string configFile, FRConfig& config) { try { if (FileExist(configFile)) { boost::property_tree::ptree pt; std::vector labels; boost::property_tree::read_json(configFile, pt); config._databasePath = GetData(pt, "database_path"); config._recEngineFile = GetData(pt, "rec_engine"); config._detEngineFile = GetData(pt, "det_engine"); return true; } else return false; } catch (std::exception& e) { std::cerr << "ANSFRHelper::LoadConfigFile" << e.what(); return false; } } void ANSFRHelper::GetFilePaths(std::string rootPath, std::vector& paths) { /* imagesPath--| |--class0--| | |--f0.jpg | |--f1.jpg | |--class1--| |--f0.jpg |--f1.jpg ... */ DIR* dir; struct dirent* entry; std::string postfix = ".jpg"; if ((dir = opendir(rootPath.c_str())) != NULL) { while ((entry = readdir(dir)) != NULL) { std::string class_path = rootPath + "/" + entry->d_name; DIR* class_dir = opendir(class_path.c_str()); struct dirent* file_entry; while ((file_entry = readdir(class_dir)) != NULL) { std::string name(file_entry->d_name); if (name.length() >= postfix.length() && 0 == name.compare(name.length() - postfix.length(), postfix.length(), postfix)) if (file_entry->d_type != DT_DIR) { struct Paths tempPaths; tempPaths.className = std::string(entry->d_name); tempPaths.absPath = class_path + "/" + name; paths.push_back(tempPaths); } } } closedir(dir); } } unsigned char* ANSCENTER::ANSFRHelper::CVMatToBytes(cv::Mat image, unsigned int& bufferLengh) { int size = int(image.total() * image.elemSize()); unsigned char* bytes = new unsigned char[size]; // you will have to delete[] that later std::memcpy(bytes, image.data, size * sizeof(unsigned char)); bufferLengh = size * sizeof(unsigned char); return bytes; } std::string ANSFRHelper::StringCurrentDateTime() { // Get the current time std::time_t t = std::time(nullptr); std::tm now = {}; // Initialize to all zeros localtime_s(&now, &t); // Convert the time to a string std::stringstream ss; ss << std::put_time(&now, "%Y%m%d%H%M%S"); std::string currentDateTime = ss.str(); return currentDateTime; } std::string ANSFRHelper::FaceObjectsToJsonString(const std::vector& faces) { boost::property_tree::ptree root; boost::property_tree::ptree detectedObjects; for (int i = 0; i < faces.size(); i++) { boost::property_tree::ptree detectedNode; detectedNode.put("user_id", faces[i].userId); detectedNode.put("user_name", faces[i].userName); detectedNode.put("similarity", faces[i].similarity); detectedNode.put("is_unknown", faces[i].isUnknown); detectedNode.put("prob", faces[i].confidence); detectedNode.put("x", faces[i].box.x); detectedNode.put("y", faces[i].box.y); detectedNode.put("width", faces[i].box.width); detectedNode.put("height", faces[i].box.height); detectedNode.put("extra_info", faces[i].extraInformation); // we might add masks into this using comma seperated string detectedObjects.push_back(std::make_pair("", detectedNode)); } root.add_child("results", detectedObjects); std::ostringstream stream; boost::property_tree::write_json(stream, root,false); std::string faceResult = stream.str(); return faceResult; } std::string ANSFRHelper::FaceAttributeToJsonString(Face face) { boost::property_tree::ptree root; boost::property_tree::ptree detectedObjects; boost::property_tree::ptree detectedNode; if(face.isAntispoofingEnabled()) { if (face.isReal()) { detectedNode.put("Antispoofing", "Real"); } else { detectedNode.put("Antispoofing", "Fake"); } } if (face.isFaceLivenessEnabled()) { if (face.getFaceLiveness() == 1) { detectedNode.put("FaceLiveness", "Real"); } else { detectedNode.put("FaceLiveness", "Fake"); } } if (face.isAgeGenderEnabled()) { if (face.isMale()) { detectedNode.put("Gender", "Male"); } else { detectedNode.put("Gender", "Female"); } int age = face.getAge(); if (age < 12) detectedNode.put("Age", "Kids"); else if (age < 20) detectedNode.put("Age", "Young Adults"); else if (age < 60) detectedNode.put("Age", "Adults"); else detectedNode.put("Age", "Elders"); } if (face.isEmotionsEnabled()) { const auto& [emotionName, emotionConfidence] = face.getMainEmotion(); std::ostringstream emotionStr; std::string adjustedEmotionName = emotionName; if (emotionName == "Sad" || emotionName == "sad") adjustedEmotionName = "Attentive"; else if (emotionName == "Happy" || emotionName == "happy") adjustedEmotionName = "Happy"; else if (emotionName == "Neutral" || emotionName == "neutral") adjustedEmotionName = "Neutral"; else if (emotionName == "Angry" || emotionName == "angry") adjustedEmotionName = "Attentive"; else if (emotionName == "Surprised" || emotionName == "surprised") adjustedEmotionName = "Attentive"; else if (emotionName == "Disgusted" || emotionName == "disgusted") adjustedEmotionName = "Attentive"; else if (emotionName == "Fearful" || emotionName == "fearful") adjustedEmotionName = "Attentive"; else if (emotionName == "Anger" || emotionName == "anger") adjustedEmotionName = "Attentive"; else adjustedEmotionName = "Neutral"; emotionStr << adjustedEmotionName << " (" << static_cast(emotionConfidence * 100) << "%)"; detectedNode.put("Emotion", emotionStr.str()); } if (face.isHeadPoseEnabled()) { const auto& headPose = face.getHeadPose(); std::ostringstream headPoseStr; headPoseStr << "(" << static_cast(headPose.angle_r) << ", " << static_cast(headPose.angle_p) << ", " << static_cast(headPose.angle_y) << ")"; detectedNode.put("Pose(r,p,y)", headPoseStr.str()); } // we might add masks into this using comma seperated string detectedObjects.push_back(std::make_pair("", detectedNode)); root.add_child("attributes", detectedObjects); std::ostringstream stream; boost::property_tree::write_json(stream, root,false); std::string faceResult = stream.str(); return faceResult; } std::string ANSFRHelper::UserRecordToJsonString(const UserRecord userRecord) { boost::property_tree::ptree root; boost::property_tree::ptree faceIds; for (int i = 0; i < userRecord.FaceIds.size(); i++) { boost::property_tree::ptree faceIdNode; faceIdNode.put("", userRecord.FaceIds[i]); faceIds.push_back(std::make_pair("", faceIdNode)); } root.put("user_id", userRecord.UserId); root.put("user_code", userRecord.UserCode); root.put("user_username", userRecord.UserName); root.add_child("face_ids", faceIds); std::ostringstream stream; boost::property_tree::write_json(stream, root,false); std::string userRecordResult = stream.str(); return userRecordResult; } std::string ANSFRHelper::UserRecordsToJsonString(const std::vector userRecords) { boost::property_tree::ptree root; boost::property_tree::ptree userRecordNodes; for (int i=0; i < userRecords.size(); i++) { boost::property_tree::ptree userRecordNode; boost::property_tree::ptree faceIds; UserRecord userRecord = userRecords[i]; for (int j = 0; j < userRecord.FaceIds.size(); j++) { boost::property_tree::ptree faceIdNode; faceIdNode.put("", userRecord.FaceIds[j]); faceIds.push_back(std::make_pair("", faceIdNode)); } userRecordNode.put("user_id", userRecord.UserId); userRecordNode.put("user_code", userRecord.UserCode); userRecordNode.put("user_username", userRecord.UserName); userRecordNode.add_child("face_ids", faceIds); userRecordNodes.push_back(std::make_pair("", userRecordNode)); } root.add_child("user_records", userRecordNodes); std::ostringstream stream; boost::property_tree::write_json(stream, root,false); std::string userRecordResult = stream.str(); return userRecordResult; } std::string ANSFRHelper::FaceRecordToJsonString(const FaceRecord faceRecord) { boost::property_tree::ptree root; root.put("face_id", faceRecord.FaceId); root.put("user_id", faceRecord.UserId); root.put("image_path", faceRecord.ImagePath); std::ostringstream stream; boost::property_tree::write_json(stream, root,false); std::string userRecordResult = stream.str(); return userRecordResult; } std::string ANSFRHelper::FaceRecordsToJsonString(const std::vector faceRecords) { boost::property_tree::ptree root; boost::property_tree::ptree faceNodes; for (int i = 0; i < faceRecords.size(); i++) { FaceRecord faceRecord = faceRecords[i]; boost::property_tree::ptree faceNode; faceNode.put("face_id", faceRecord.FaceId); faceNode.put("user_id", faceRecord.UserId); faceNode.put("image_path", faceRecord.ImagePath); faceNodes.push_back(std::make_pair("", faceNode)); } root.add_child("face_records", faceNodes); std::ostringstream stream; boost::property_tree::write_json(stream, root,false); std::string userRecordResult = stream.str(); return userRecordResult; } cv::Mat ANSFRHelper::PreprocessImg(cv::Mat& img, int input_w, int input_h) { try { int w, h, x, y; float r_w = input_w / (img.cols * 1.0); float r_h = input_h / (img.rows * 1.0); if (r_h > r_w) { w = input_w; h = r_w * img.rows; x = 0; y = (input_h - h) / 2; } else { w = r_h * img.cols; h = input_h; x = (input_w - w) / 2; y = 0; } cv::Mat re(h, w, CV_8UC3); cv::resize(img, re, re.size(), 0, 0, cv::INTER_LINEAR); cv::Mat out(input_h, input_w, CV_8UC3, cv::Scalar(128, 128, 128)); re.copyTo(out(cv::Rect(x, y, re.cols, re.rows))); return out; } catch (std::exception& e) { cv::Mat result; std::cerr << "ANSFRHelper::PreprocessImg" << e.what(); return result; } } int ANSFRHelper::ReadFilesInDir(const char* p_dir_name, std::vector& file_names) { try { DIR* p_dir = opendir(p_dir_name); if (p_dir == nullptr) { return -1; } struct dirent* p_file = nullptr; while ((p_file = readdir(p_dir)) != nullptr) { if (strcmp(p_file->d_name, ".") != 0 && strcmp(p_file->d_name, "..") != 0) { std::string cur_file_name(p_file->d_name); file_names.push_back(cur_file_name); } } closedir(p_dir); return 0; } catch (std::exception& e) { std::cerr << "ANSFRHelper::ReadFilesInDir" << e.what(); return -1; } } cv::Rect ANSFRHelper::GetRectAdaptLandmark(cv::Mat& img, int input_w, int input_h, float bbox[4], float lmk[10]) { int l, r, t, b; float r_w = input_w / (img.cols * 1.0); float r_h = input_h / (img.rows * 1.0); if (r_h > r_w) { l = bbox[0] / r_w; r = bbox[2] / r_w; t = (bbox[1] - (input_h - r_w * img.rows) / 2) / r_w; b = (bbox[3] - (input_h - r_w * img.rows) / 2) / r_w; for (int i = 0; i < 10; i += 2) { lmk[i] /= r_w; lmk[i + 1] = (lmk[i + 1] - (input_h - r_w * img.rows) / 2) / r_w; } } else { l = (bbox[0] - (input_w - r_h * img.cols) / 2) / r_h; r = (bbox[2] - (input_w - r_h * img.cols) / 2) / r_h; t = bbox[1] / r_h; b = bbox[3] / r_h; for (int i = 0; i < 10; i += 2) { lmk[i] = (lmk[i] - (input_w - r_h * img.cols) / 2) / r_h; lmk[i + 1] /= r_h; } } return cv::Rect(l, t, r - l, b - t); } float ANSFRHelper::IOU(float lbox[4], float rbox[4]) { float interBox[] = { MAX(lbox[0], rbox[0]), //left MIN(lbox[2], rbox[2]), //right MAX(lbox[1], rbox[1]), //top MIN(lbox[3], rbox[3]), //bottom }; if (interBox[2] > interBox[3] || interBox[0] > interBox[1]) return 0.0f; float interBoxS = (interBox[1] - interBox[0]) * (interBox[3] - interBox[2]); return interBoxS / ((lbox[2] - lbox[0]) * (lbox[3] - lbox[1]) + (rbox[2] - rbox[0]) * (rbox[3] - rbox[1]) - interBoxS + 0.000001f); } bool ANSFRHelper::CMP(const Detection& a, const Detection& b) { return a.class_confidence > b.class_confidence; } void ANSFRHelper::NMS(std::vector& res, float* output, float nms_thresh) { std::vector dets; for (int i = 0; i < output[0]; i++) { if (output[15 * i + 1 + 4] <= 0.1) continue; Detection det; memcpy(&det, &output[15 * i + 1], sizeof(Detection)); dets.push_back(det); } std::sort(dets.begin(), dets.end(), ANSFRHelper::CMP); for (size_t m = 0; m < dets.size(); ++m) { auto& item = dets[m]; res.push_back(item); //std::cout << item.class_confidence << " bbox " << item.bbox[0] << ", " << item.bbox[1] << ", " << item.bbox[2] << ", " << item.bbox[3] << std::endl; for (size_t n = m + 1; n < dets.size(); ++n) { if (ANSFRHelper::IOU(item.bbox, dets[n].bbox) > nms_thresh) { dets.erase(dets.begin() + n); --n; } } } } std::map ANSFRHelper::LoadWeights(const std::string file) { std::cout << "Loading weights: " << file << std::endl; std::map weightMap; // Open weights file std::ifstream input(file); assert(input.is_open() && "Unable to load weight file."); // Read number of weight blobs int32_t count; input >> count; assert(count > 0 && "Invalid weight map file."); while (count--) { Weights wt{ DataType::kFLOAT, nullptr, 0 }; uint32_t size; // Read name and type of blob std::string name; input >> name >> std::dec >> size; wt.type = DataType::kFLOAT; // Load blob uint32_t* val = reinterpret_cast(malloc(sizeof(val) * size)); for (uint32_t x = 0, y = size; x < y; ++x) { input >> std::hex >> val[x]; } wt.values = val; wt.count = size; weightMap[name] = wt; } return weightMap; } Weights ANSFRHelper::GetWeights(std::map& weightMap, std::string key) { if (weightMap.count(key) != 1) { std::cerr << key << " not existed in weight map, fatal error!!!" << std::endl; exit(-1); } return weightMap[key]; } IScaleLayer* ANSFRHelper::AddBatchNorm2D(INetworkDefinition* network, std::map& weightMap, ITensor& input, std::string lname, float eps) { float* gamma = (float*)weightMap[lname + ".weight"].values; float* beta = (float*)weightMap[lname + ".bias"].values; float* mean = (float*)weightMap[lname + ".running_mean"].values; float* var = (float*)weightMap[lname + ".running_var"].values; int len = weightMap[lname + ".running_var"].count; float* scval = reinterpret_cast(malloc(sizeof(float) * len)); for (int i = 0; i < len; i++) { scval[i] = gamma[i] / sqrt(var[i] + eps); } Weights scale{ DataType::kFLOAT, scval, len }; float* shval = reinterpret_cast(malloc(sizeof(float) * len)); for (int i = 0; i < len; i++) { shval[i] = beta[i] - mean[i] * gamma[i] / sqrt(var[i] + eps); } Weights shift{ DataType::kFLOAT, shval, len }; float* pval = reinterpret_cast(malloc(sizeof(float) * len)); for (int i = 0; i < len; i++) { pval[i] = 1.0; } Weights power{ DataType::kFLOAT, pval, len }; weightMap[lname + ".scale"] = scale; weightMap[lname + ".shift"] = shift; weightMap[lname + ".power"] = power; IScaleLayer* scale_1 = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power); assert(scale_1); return scale_1; } /// /// MatMul /// //MatMul::MatMul() { // try { // ANSFRHelper::ANSFRHelper::CheckCudaStatus(cudaMalloc(&workspace, workspaceSize)); // ANSFRHelper::CheckCublasStatus(cublasLtCreate(<Handle)); // ANSFRHelper::CheckCudaStatus(cudaStreamCreate(&stream)); // } // catch (std::exception& e) { // std::cerr << "MatMul::MatMul" << e.what(); // } //} //void MatMul::Init(float* knownEmbeds, int numRow, int numCol) { // try { // m = static_cast(numRow); // k = static_cast(numCol); // lda = static_cast(numCol); // ldb = static_cast(numCol); // ldc = static_cast(numRow); // // alloc and copy known embeddings to GPU // ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast(&dA), m * k * sizeof(float))); // ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(dA, knownEmbeds, m * k * sizeof(float), cudaMemcpyHostToDevice, stream)); // // create operation desciriptor; see cublasLtMatmulDescAttributes_t for details about defaults; // // here we just need to set the transforms for A and B // ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescCreate(&operationDesc, computeType, cudaDataType)); // ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa))); // ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transb))); // // create matrix descriptors, we are good with the details here so no need to set any extra attributes // ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Adesc, cudaDataType, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda)); // // create preference handle; here we could use extra attributes to disable tensor ops or to make sure algo selected // // will work with badly aligned A, B, C; here for simplicity we just assume A,B,C are always well aligned (e.g. // // directly come from cudaMalloc) // ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceCreate(&preference)); // ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceSetAttribute(preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize))); // } // catch (std::exception& e) { // std::cerr << "MatMul::Init" << e.what(); // } //} //void MatMul::Calculate(float* embeds, int embedCount, float* outputs) { // try { // n = embedCount; // // Allocate arrays on GPU // ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast(&dB), k * n * sizeof(float))); // ANSFRHelper::CheckCudaStatus(cudaMalloc(reinterpret_cast(&dC), m * n * sizeof(float))); // ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(dB, embeds, k * n * sizeof(float), cudaMemcpyHostToDevice, stream)); // // create matrix descriptors, we are good with the details here so no need to set any extra attributes // cublasLtMatrixLayout_t Bdesc = NULL, Cdesc = NULL; // ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Bdesc, cudaDataType, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb)); // ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutCreate(&Cdesc, cudaDataType, m, n, ldc)); // // we just need the best available heuristic to try and run matmul. There is no guarantee this will work, e.g. if A // // is badly aligned, you can request more (e.g. 32) algos and try to run them one by one until something works // int returnedResults = 0; // cublasLtMatmulHeuristicResult_t heuristicResult = {}; // ANSFRHelper::CheckCublasStatus(cublasLtMatmulAlgoGetHeuristic(ltHandle, operationDesc, Adesc, Bdesc, Cdesc, Cdesc, preference, 1, &heuristicResult, &returnedResults)); // if (returnedResults == 0) { // ANSFRHelper::CheckCublasStatus(CUBLAS_STATUS_NOT_SUPPORTED); // } // // Do the actual multiplication // ANSFRHelper::CheckCublasStatus(cublasLtMatmul(ltHandle, operationDesc, &alpha, dA, Adesc, dB, Bdesc, &beta, dC, Cdesc, dC, Cdesc, &heuristicResult.algo, workspace, // workspaceSize, stream)); // // Cleanup: descriptors are no longer needed as all GPU work was already enqueued // if (Cdesc) // ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Cdesc)); // if (Bdesc) // ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Bdesc)); // // Copy the result on host memory // ANSFRHelper::CheckCudaStatus(cudaMemcpyAsync(outputs, dC, m * n * sizeof(float), cudaMemcpyDeviceToHost, stream)); // // CUDA stream sync // ANSFRHelper::CheckCudaStatus(cudaStreamSynchronize(stream)); // // Free GPU memory // ANSFRHelper::CheckCudaStatus(cudaFree(dB)); // ANSFRHelper::CheckCudaStatus(cudaFree(dC)); // } // catch (std::exception& e) { // std::cerr <<"MatMul::Calculate"<< e.what(); // } //} //MatMul::~MatMul() { // try { // if (preference) // ANSFRHelper::CheckCublasStatus(cublasLtMatmulPreferenceDestroy(preference)); // if (Adesc) // ANSFRHelper::CheckCublasStatus(cublasLtMatrixLayoutDestroy(Adesc)); // if (operationDesc) // ANSFRHelper::CheckCublasStatus(cublasLtMatmulDescDestroy(operationDesc)); // ANSFRHelper::CheckCublasStatus(cublasLtDestroy(ltHandle)); // ANSFRHelper::CheckCudaStatus(cudaFree(dA)); // ANSFRHelper::CheckCudaStatus(cudaFree(workspace)); // ANSFRHelper::CheckCudaStatus(cudaStreamDestroy(stream)); // } // catch (std::exception& e) { // std::cout << "MatMul::~MatMul" << e.what(); // } //} //Int8EntropyCalibrator2::Int8EntropyCalibrator2(int batchsize, int input_w, int input_h, const char* img_dir, const char* calib_table_name, const char* input_blob_name, bool read_cache) // : batchsize_(batchsize) // , input_w_(input_w) // , input_h_(input_h) // , img_idx_(0) // , img_dir_(img_dir) // , calib_table_name_(calib_table_name) // , input_blob_name_(input_blob_name) // , read_cache_(read_cache) //{ // input_count_ = 3 * input_w * input_h * batchsize; // CUDACHECK(cudaMalloc(&device_input_, input_count_ * sizeof(float))); // ANSFRHelper::ReadFilesInDir(img_dir, img_files_); //} //Int8EntropyCalibrator2::~Int8EntropyCalibrator2() //{ // CUDACHECK(cudaFree(device_input_)); //} //int Int8EntropyCalibrator2::getBatchSize() const TRT_NOEXCEPT //{ // return batchsize_; //} //bool Int8EntropyCalibrator2::getBatch(void* bindings[], const char* names[], int nbBindings) TRT_NOEXCEPT //{ // if (img_idx_ + batchsize_ > (int)img_files_.size()) { // return false; // } // std::vector input_imgs_; // for (int i = img_idx_; i < img_idx_ + batchsize_; i++) { // std::cout << img_files_[i] << " " << i << std::endl; // cv::Mat temp = cv::imread(img_dir_ + img_files_[i]); // if (temp.empty()) { // std::cerr << "Fatal error: image cannot open!" << std::endl; // return false; // } // cv::Mat pr_img = ANSFRHelper::PreprocessImg(temp, input_w_, input_h_); // input_imgs_.push_back(pr_img); // } // img_idx_ += batchsize_; // cv::Mat blob = cv::dnn::blobFromImages(input_imgs_, 1.0, cv::Size(input_w_, input_h_), cv::Scalar(104, 117, 123), false, false); // CUDACHECK(cudaMemcpy(device_input_, blob.ptr(0), input_count_ * sizeof(float), cudaMemcpyHostToDevice)); // assert(!strcmp(names[0], input_blob_name_)); // bindings[0] = device_input_; // return true; //} //const void* Int8EntropyCalibrator2::readCalibrationCache(size_t& length) TRT_NOEXCEPT //{ // std::cout << "reading calib cache: " << calib_table_name_ << std::endl; // calib_cache_.clear(); // std::ifstream input(calib_table_name_, std::ios::binary); // input >> std::noskipws; // if (read_cache_ && input.good()) // { // std::copy(std::istream_iterator(input), std::istream_iterator(), std::back_inserter(calib_cache_)); // } // length = calib_cache_.size(); // return length ? calib_cache_.data() : nullptr; //} //void Int8EntropyCalibrator2::writeCalibrationCache(const void* cache, size_t length) TRT_NOEXCEPT //{ // std::cout << "writing calib cache: " << calib_table_name_ << " size: " << length << std::endl; // std::ofstream output(calib_table_name_, std::ios::binary); // output.write(reinterpret_cast(cache), length); //} }