Refactor project structure

This commit is contained in:
2026-03-28 19:56:39 +11:00
parent 1d267378b2
commit 8a2e721058
511 changed files with 59 additions and 48 deletions
+695
View File
@@ -0,0 +1,695 @@
#include "ANSANOMALIB.h"
namespace ANSCENTER {
bool ANSANOMALIB::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("ANSANOMALIB::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("ANSANOMALIB::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("ANSANOMALIB::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool ANSANOMALIB::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_device = "AUTO:GPU,CPU";
_openvinoPreprocess = true;
_efficientAd = false;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlfile = CreateFilePath(_modelFolder, "model.xml");
std::string metafile = CreateFilePath(_modelFolder, "metadata.json");
std::string binaryfile = CreateFilePath(_modelFolder, "model.bin");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
_metaFilePath = metafile;
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(_metaFilePath)) {
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading meta data", _metaFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading binary model", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Binary model file does not exit", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Meta data file does not exist", _metaFilePath, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Model file does not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
// Load Model from Here
InitModel(_modelFilePath, _metaFilePath, _device, _openvinoPreprocess, _efficientAd);
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSANOMALIB::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "model";
}
std::string xmlfileName = _modelName + ".xml";
std::string binfileName = _modelName + ".bin";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::SEGMENTATION;
_modelConfig.modelType = ModelType::ANOMALIB;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
_openvinoPreprocess = true;
_efficientAd = false;
_device = "AUTO:GPU,CPU";
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlfile = CreateFilePath(_modelFolder, xmlfileName);
std::string metafile = CreateFilePath(_modelFolder, "metadata.json");
std::string binaryfile = CreateFilePath(_modelFolder, binfileName);
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
_metaFilePath = metafile;
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(_metaFilePath)) {
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading meta data", _metaFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading binary model", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Binary model file does not exit", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Meta data file does not exist", _metaFilePath, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Model file does not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
// Load Model from Here
InitModel(_modelFilePath, _metaFilePath, _device, _openvinoPreprocess, _efficientAd);
labelMap = "Normal,Defect";
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSANOMALIB::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
std::string openVINOVersion = ov::get_openvino_version().buildNumber;
//this->_logger.LogDebug("ANSANOMALIB::Initialize. OpenVINO version", openVINOVersion, __FILE__, __LINE__);
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for ABNORMAL only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::SEGMENTATION;
_modelConfig.modelType = ModelType::ANOMALIB;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
_openvinoPreprocess = true;
_efficientAd = false;
_device = "AUTO:GPU,CPU";
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlfile = CreateFilePath(_modelFolder, "model.xml");
std::string metafile = CreateFilePath(_modelFolder, "metadata.json");
std::string binaryfile = CreateFilePath(_modelFolder, "model.bin");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
_metaFilePath = metafile;
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(_metaFilePath)) {
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading meta data", _metaFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSANOMALIB::Initialize. Loading binary model", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Binary model file does not exit", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Meta data file does not exist", _metaFilePath, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSANOMALIB::Initialize. Model file does not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
// Load Model from Here
InitModel(_modelFilePath, _metaFilePath, _device, _openvinoPreprocess, _efficientAd);
labelMap = "Normal,Defect";
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::Initialize", e.what(), __FILE__, __LINE__);
labelMap = "Normal,Defect";
return false;
}
}
std::vector<Object> ANSANOMALIB::RunInference(const cv::Mat& input) {
return RunInference(input, "ANSANOMALIBCAM");
}
std::vector<Object> ANSANOMALIB::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<Object> output;
output.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSANOMALIB::RunInference", "Invalid License", __FILE__, __LINE__);
return output;
}
if (!_isInitialized) {
this->_logger.LogError("ANSANOMALIB::RunInference", "Model is not initialized", __FILE__, __LINE__);
return output;
}
try {
if (input.empty()) return output;
if ((input.cols < 10) || (input.rows < 10)) return output;
cv::Mat frame = input;
Object result = InferenceSingleImage(frame);
result.cameraId = camera_id;
output.push_back(result);
//EnqueueDetection(output, camera_id);
return output;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::RunInference", e.what(), __FILE__, __LINE__);
return output;
}
}
ANSANOMALIB::~ANSANOMALIB() {
try {
//this->_logger.LogDebug("ANSANOMALIB::~ANSANOMALIB()", "Release ANSANOMALIB ", __FILE__, __LINE__);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::~ANSANOMALIB()", e.what(), __FILE__, __LINE__);
}
}
bool ANSANOMALIB::Destroy() {
try {
//this->_logger.LogDebug("ANSANOMALIB::Destroy", "Release ANSANOMALIB ", __FILE__, __LINE__);
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
// Private
void ANSANOMALIB::InitModel(std::string& model_path,
std::string& meta_path,
std::string& device,
bool openvino_preprocess,
bool efficient_ad) {
try {
this->_efficientAd = efficient_ad;
this->_openvinoPreprocess = openvino_preprocess;
this->_meta = ANSUtilityHelper::GetJson(meta_path);
this->GetModel(model_path, device);
this->_inputs = this->_compiledModel.inputs();
this->_outputs = this->_compiledModel.outputs();
for (auto input : this->_inputs) {
std::cout << "Input: " << input.get_any_name() << ": [ ";
for (auto j : input.get_shape()) {
std::cout << j << " ";
}
std::cout << "] ";
std::cout << "dtype: " << input.get_element_type() << std::endl;
}
for (auto output : this->_outputs) {
std::cout << "Output: " << output.get_any_name() << ": [ ";
for (auto j : output.get_shape()) {
std::cout << j << " ";
}
std::cout << "] ";
std::cout << "dtype: " << output.get_element_type() << std::endl;
}
this->_inferRequest = this->_compiledModel.create_infer_request();
//this->WarmUp();
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::InitModel", e.what(), __FILE__, __LINE__);
}
}
void ANSANOMALIB::GetModel(std::string& model_path, std::string& device) {
try {
// Step 1. Initialize OpenVINO Runtime core
ov::Core core;
// Step 2. Read a Model from a Drive
std::shared_ptr<ov::Model> model = core.read_model(model_path);
if (this->_openvinoPreprocess) {
std::vector<float> mean;
std::vector<float> std;
if (!this->_efficientAd) {
mean = { this->_meta._mean[0] * 255, this->_meta._mean[1] * 255, this->_meta._mean[2] * 255 };
std = { this->_meta._std[0] * 255, this->_meta._std[1] * 255, this->_meta._std[2] * 255 };
}
else {
mean = { 0., 0., 0. };
std = { 255., 255., 255. };
}
// Step 3. Inizialize Preprocessing for the model
// https://mp.weixin.qq.com/s/4lkDJC95at2tK_Zd62aJxw
// https://blog.csdn.net/sandmangu/article/details/107181289
// https://docs.openvino.ai/latest/openvino_2_0_preprocessing.html
ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model);
// Specify input image format
ppp.input(0).tensor()
.set_color_format(ov::preprocess::ColorFormat::RGB) // BGR -> RGB
.set_element_type(ov::element::u8)
.set_layout(ov::Layout("HWC")); // HWC NHWC NCHW
// Specify preprocess pipeline to input image without resizing
ppp.input(0).preprocess()
// .convert_color(ov::preprocess::ColorFormat::RGB)
.convert_element_type(ov::element::f32)
.mean(mean)
.scale(std);
// Specify model's input layout
ppp.input(0).model().set_layout(ov::Layout("NCHW"));
// Specify output results format
for (size_t i = 0; i < model->outputs().size(); i++) {
ppp.output(i).tensor().set_element_type(ov::element::f32);
}
// Embed above steps in the graph
model = ppp.build();
}
// Step 4. Load the Model to the Device
this->_compiledModel = core.compile_model(model, device,
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::inference_num_threads(4));
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::GetModel", e.what(), __FILE__, __LINE__);
}
}
void ANSANOMALIB::WarmUp() {
try {
cv::Size size = cv::Size(this->_meta.inferSize[1], this->_meta.inferSize[0]);
cv::Scalar color = cv::Scalar(0, 0, 0);
cv::Mat input = cv::Mat(size, CV_8UC3, color);
this->Inference(input);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::WarmUp", e.what(), __FILE__, __LINE__);
}
}
Object ANSANOMALIB::Inference(cv::Mat& image) {
try {
Object result;
cv::Mat blob;
if (image.empty())return result;
if ((image.cols <= 5) || (image.rows <= 5))return result;
cv::Mat processedImage;
if (image.channels() == 1) {
cv::cvtColor(image, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = image;
}
this->_meta.imageSize[0] = processedImage.size().height;
this->_meta.imageSize[1] = processedImage.size().width;
cv::cvtColor(processedImage, processedImage, cv::ColorConversionCodes::COLOR_BGR2RGB); // BGR2RGB
if (this->_openvinoPreprocess) {
cv::resize(processedImage, blob, { this->_meta.inferSize[0], this->_meta.inferSize[1] });
}
else {
blob = PreProcess(processedImage, this->_meta, this->_efficientAd);
blob = cv::dnn::blobFromImage(blob); // [H, W, C] -> [N, C, H, W]
}
ov::Tensor input_tensor = ov::Tensor(this->_compiledModel.input(0).get_element_type(),
this->_compiledModel.input(0).get_shape(), (float*)blob.data);
this->_inferRequest.set_input_tensor(input_tensor);
this->_inferRequest.infer();
ov::Tensor result1;
result1 = this->_inferRequest.get_output_tensor(0);
cv::Mat anomaly_map = cv::Mat(cv::Size(this->_meta.inferSize[1], this->_meta.inferSize[0]), CV_32FC1, result1.data<float>());
cv::Mat pred_score;
if (this->_outputs.size() == 2) {
ov::Tensor result2 = this->_inferRequest.get_output_tensor(1);
pred_score = cv::Mat(cv::Size(1, 1), CV_32FC1, result2.data<float>());
}
else {
double _, maxValue;
cv::minMaxLoc(anomaly_map, &_, &maxValue);
pred_score = cv::Mat(cv::Size(1, 1), CV_32FC1, maxValue);
}
float preScore = pred_score.at<float>(0, 0);
std::vector<cv::Mat> post_mat = PostProcess(anomaly_map, pred_score, this->_meta);
anomaly_map = post_mat[0];
float rawScore = post_mat[1].at<float>(0, 0);
float score = rawScore;
if (preScore >= this->_meta.imageThreshold) {
result.classId = 1;
result.className = "Defect";
}
else {
result.classId = 0;
score = 1 - rawScore;
result.className = "Normal";
}
result.box.x = 5;
result.box.y = 5;
result.box.width = this->_meta.imageSize[0] - 5;
result.box.height = this->_meta.imageSize[1] - 5;
result.polygon = ANSUtilityHelper::RectToNormalizedPolygon(result.box, processedImage.cols, processedImage.rows);
result.confidence = score;
result.mask = anomaly_map;
result.extraInfo = "";
return result;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::Inference", e.what(), __FILE__, __LINE__);
Object result;
return result;
}
}
Object ANSANOMALIB::InferenceSingleImage(cv::Mat& image) {
try {
Object result = this->Inference(image);
std::vector<cv::Mat> images = GenImages(image, result.mask, result.confidence);
cv::applyColorMap(images[0], images[0], cv::ColormapTypes::COLORMAP_JET);
cv::Mat res;
cv::hconcat(images, res);
result.mask = res;
return result;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::InferenceSingleImage", e.what(), __FILE__, __LINE__);
Object result;
return result;
}
}
void ANSANOMALIB::InferenceMultiImages(std::string& image_dir, std::string& save_dir) {
try {
std::vector<cv::String> paths = GetImagePaths(image_dir);
for (auto& image_path : paths) {
cv::Mat image = ReadImage(image_path);
Object result = this->Inference(image);
std::vector<cv::Mat> images = GenImages(image, result.mask, result.confidence);
cv::applyColorMap(images[0], images[0], cv::ColormapTypes::COLORMAP_JET);
cv::Mat res;
cv::hconcat(images, res);
SaveScoreAndImages(result.confidence, res, image_path, save_dir);
}
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::InferenceMultiImages", e.what(), __FILE__, __LINE__);
}
}
std::vector<cv::String> ANSANOMALIB::GetImagePaths(std::string& path) {
try {
std::vector<cv::String> paths;
cv::glob(path, paths, false);
return paths;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::GetImagePaths", e.what(), __FILE__, __LINE__);
std::vector<cv::String> paths;
paths.clear();
return paths;
}
}
cv::Mat ANSANOMALIB::ReadImage(std::string& path) {
try {
auto image = cv::imread(path, cv::ImreadModes::IMREAD_COLOR);
cv::cvtColor(image, image, cv::ColorConversionCodes::COLOR_BGR2RGB);
return image;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::ReadImage", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
void ANSANOMALIB::SaveScoreAndImages(float score, cv::Mat image,
cv::String& image_path, std::string& save_dir) {
try {
auto start = image_path.rfind('\\');
if (start < 0 || start > image_path.length()) {
start = image_path.rfind('/');
}
auto end = image_path.substr(start + 1).rfind('.');
auto image_name = image_path.substr(start + 1).substr(0, end);
std::ofstream ofs;
ofs.open(save_dir + "/" + image_name + ".txt", std::ios::out);
ofs << score;
ofs.close();
cv::imwrite(save_dir + "/" + image_name + ".jpg", image);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::SaveScoreAndImages", e.what(), __FILE__, __LINE__);
}
}
cv::Mat ANSANOMALIB::PreProcess(cv::Mat& image, MetaData& meta, bool efficient_ad) {
try {
std::vector<float> mean = { 0.485, 0.456, 0.406 };
std::vector<float> std = { 0.229, 0.224, 0.225 };
cv::Mat resized_image = ANSUtilityHelper::Resize(image, meta.inferSize[0], meta.inferSize[1], "bilinear");
resized_image.convertTo(resized_image, CV_32FC3, 1.0 / 255, 0);
if (!efficient_ad) {
resized_image = ANSUtilityHelper::Normalize(resized_image, mean, std);
}
return resized_image;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::PreProcess", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
cv::Mat ANSANOMALIB::NormalizeMinMax(cv::Mat& targets, float threshold, float min_val, float max_val) {
try {
cv::Mat normalized = ((targets - threshold) / (max_val - min_val)) + 0.5;
cv::threshold(normalized, normalized, 1, 1, cv::ThresholdTypes::THRESH_TRUNC);
cv::threshold(normalized, normalized, 0, 0, cv::ThresholdTypes::THRESH_TOZERO);
return normalized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::NormalizeMinMax", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
std::vector<cv::Mat> ANSANOMALIB::PostProcess(cv::Mat& anomaly_map, cv::Mat& pred_score, MetaData& meta) {
try {
anomaly_map = NormalizeMinMax(anomaly_map, meta.pixelThreshold, meta.min, meta.max);
pred_score = NormalizeMinMax(pred_score, meta.imageThreshold, meta.min, meta.max);
anomaly_map = ANSUtilityHelper::Resize(anomaly_map, meta.imageSize[0], meta.imageSize[1], "bilinear");
return std::vector<cv::Mat>{anomaly_map, pred_score};
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::PostProcess", e.what(), __FILE__, __LINE__);
std::vector<cv::Mat> result;
result.clear();
return result;
}
}
cv::Mat ANSANOMALIB::SuperimposeAnomalyMap(cv::Mat& anomaly_map, cv::Mat& origin_image) {
try {
auto anomaly = anomaly_map.clone();
double minValue, maxValue;
cv::minMaxLoc(anomaly, &minValue, &maxValue);
anomaly = (anomaly - minValue) / (maxValue - minValue);
anomaly.convertTo(anomaly, CV_8UC1, 255, 0);
cv::applyColorMap(anomaly, anomaly, cv::ColormapTypes::COLORMAP_JET);
cv::Mat combine;
cv::addWeighted(anomaly, 0.4, origin_image, 0.6, 0, combine);
return combine;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::SuperimposeAnomalyMap", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
cv::Mat ANSANOMALIB::AddLabel(cv::Mat& mixed_image, float score, int font) {
try {
std::string text = "Confidence Score " + std::to_string(score);
int font_size = mixed_image.cols / 1024 + 1;
int baseline = 0;
int thickness = font_size / 2;
cv::Size textsize = cv::getTextSize(text, font, font_size, thickness, &baseline);
cv::rectangle(mixed_image, cv::Point(0, 0), cv::Point(textsize.width + 10, textsize.height + 10),
cv::Scalar(225, 252, 134), cv::FILLED);
cv::putText(mixed_image, text, cv::Point(0, textsize.height + 10), font, font_size,
cv::Scalar(0, 0, 0), thickness);
return mixed_image;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::AddLabel", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
cv::Mat ANSANOMALIB::ComputeMask(cv::Mat& anomaly_map, float threshold, int kernel_size) {
try {
cv::Mat mask = anomaly_map.clone();
cv::threshold(mask, mask, threshold, 1, cv::ThresholdTypes::THRESH_BINARY);
auto kernel = cv::getStructuringElement(cv::MorphShapes::MORPH_ELLIPSE, { kernel_size, kernel_size }, { -1, -1 });
cv::morphologyEx(mask, mask, cv::MorphTypes::MORPH_OPEN, kernel, { -1, -1 }, 1);
mask.convertTo(mask, CV_8UC1, 255, 0);
return mask;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::ComputeMask", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
cv::Mat ANSANOMALIB::GenMaskBorder(cv::Mat& mask, cv::Mat& image) {
try {
cv::Mat b = cv::Mat::zeros(mask.size[0], mask.size[1], CV_8UC1);
cv::Mat g = b.clone();
cv::Mat r = b.clone();
cv::Canny(mask, r, 128, 255, 3, false);
std::vector<cv::Mat> rgb{ b, g, r };
cv::Mat border;
cv::merge(rgb, border);
cv::addWeighted(border, 0.4, image, 0.6, 0, border);
return border;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::GenMaskBorder", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
std::vector<cv::Mat> ANSANOMALIB::GenImages(cv::Mat& image, cv::Mat& anomaly_map, float score, float threshold) {
try {
cv::cvtColor(image, image, cv::ColorConversionCodes::COLOR_RGB2BGR);
cv::Mat mask = ComputeMask(anomaly_map, threshold);
cv::Mat border = GenMaskBorder(mask, image);
cv::Mat superimposed_map = SuperimposeAnomalyMap(anomaly_map, image);
superimposed_map = AddLabel(superimposed_map, score);
return std::vector<cv::Mat>{mask, border, superimposed_map};
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::GenImages", e.what(), __FILE__, __LINE__);
cv::Mat result;
return result;
}
}
}
+55
View File
@@ -0,0 +1,55 @@
#ifndef ANSANOMALIB_H
#define ANSANOMALIB_H
#pragma once
#include "ANSEngineCommon.h"
#include <opencv2/opencv.hpp>
#include <opencv2/dnn.hpp>
#include <openvino/openvino.hpp>
namespace ANSCENTER
{
class ANSENGINE_API ANSANOMALIB :public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSANOMALIB();
private:
bool _openvinoPreprocess;
bool _efficientAd;
MetaData _meta{};
ov::CompiledModel _compiledModel;
ov::InferRequest _inferRequest;
std::vector<ov::Output<const ov::Node>> _inputs;
std::vector<ov::Output<const ov::Node>> _outputs;
std::string _modelFilePath;
std::string _metaFilePath;
std::string _device = "CPU";
private:
void InitModel(std::string& model_path, std::string& meta_path, std::string& device, bool openvinoPreprocess, bool efficientAd = false);
void GetModel(std::string& model_path, std::string& device);
void WarmUp();
Object Inference(cv::Mat& image);
Object InferenceSingleImage(cv::Mat& image);
void InferenceMultiImages(std::string& imageDir, std::string& saveDir);
std::vector<cv::String> GetImagePaths(std::string& path);
cv::Mat ReadImage(std::string& path);
void SaveScoreAndImages(float score, cv::Mat image, cv::String& imagePath, std::string& saveDir);
cv::Mat PreProcess(cv::Mat& image, MetaData& meta, bool efficientAd = false);
cv::Mat NormalizeMinMax(cv::Mat& targets, float threshold, float minVal, float maxVal);
std::vector<cv::Mat> PostProcess(cv::Mat& anomalyMap, cv::Mat& predScore, MetaData& meta);
cv::Mat SuperimposeAnomalyMap(cv::Mat& anomalyMap, cv::Mat& originImage);
cv::Mat AddLabel(cv::Mat& mixedImage, float score, int font = cv::FONT_HERSHEY_PLAIN);
cv::Mat ComputeMask(cv::Mat& anomalyMap, float threshold = 0.5, int kernelSize = 1);
cv::Mat GenMaskBorder(cv::Mat& mask, cv::Mat& image);
std::vector<cv::Mat> GenImages(cv::Mat& image, cv::Mat& anomalyMap, float score, float threshold = 0.5);
};
}
#endif
+286
View File
@@ -0,0 +1,286 @@
#include "ANSCLIPTokenizer.h"
#include <fstream>
#include <algorithm>
#include <climits>
#include <cctype>
#include <set>
namespace ANSCENTER
{
// =========================================================================
// UTF-8 helper
// =========================================================================
std::string ANSCLIPTokenizer::codepointToUtf8(int cp)
{
std::string s;
if (cp < 0x80) {
s += static_cast<char>(cp);
}
else if (cp < 0x800) {
s += static_cast<char>(0xC0 | (cp >> 6));
s += static_cast<char>(0x80 | (cp & 0x3F));
}
else if (cp < 0x10000) {
s += static_cast<char>(0xE0 | (cp >> 12));
s += static_cast<char>(0x80 | ((cp >> 6) & 0x3F));
s += static_cast<char>(0x80 | (cp & 0x3F));
}
return s;
}
// =========================================================================
// Byte encoder - CLIP's bytes_to_unicode()
// =========================================================================
void ANSCLIPTokenizer::initByteEncoder()
{
// 188 "printable" bytes map to their own code point.
// The remaining 68 bytes map to code points 256..323.
std::set<int> printable;
for (int i = 33; i <= 126; ++i) printable.insert(i); // ! to ~
for (int i = 161; i <= 172; ++i) printable.insert(i); // inverted-! to not-sign
for (int i = 174; i <= 255; ++i) printable.insert(i); // registered-sign to y-umlaut
int extra = 256;
for (int b = 0; b < 256; ++b) {
int cp = printable.count(b) ? b : extra++;
m_byteEncoder[b] = codepointToUtf8(cp);
}
}
// =========================================================================
// Pre-tokenisation (simplified CLIP regex for ASCII text)
// =========================================================================
std::vector<std::string> ANSCLIPTokenizer::preTokenize(const std::string& text)
{
// Lowercase
std::string lower;
lower.reserve(text.size());
for (unsigned char c : text)
lower += static_cast<char>(std::tolower(c));
std::vector<std::string> tokens;
size_t i = 0;
while (i < lower.size()) {
// Skip whitespace
if (std::isspace(static_cast<unsigned char>(lower[i]))) {
++i;
continue;
}
// Contractions: 's 't 'm 'd 're 've 'll
if (lower[i] == '\'' && i + 1 < lower.size()) {
char c2 = lower[i + 1];
if (c2 == 's' || c2 == 't' || c2 == 'm' || c2 == 'd') {
tokens.push_back(lower.substr(i, 2));
i += 2;
continue;
}
if (i + 2 < lower.size()) {
std::string tri = lower.substr(i, 3);
if (tri == "'re" || tri == "'ve" || tri == "'ll") {
tokens.push_back(tri);
i += 3;
continue;
}
}
}
// Letters
if (std::isalpha(static_cast<unsigned char>(lower[i]))) {
size_t start = i;
while (i < lower.size() && std::isalpha(static_cast<unsigned char>(lower[i])))
++i;
tokens.push_back(lower.substr(start, i - start));
continue;
}
// Digits
if (std::isdigit(static_cast<unsigned char>(lower[i]))) {
size_t start = i;
while (i < lower.size() && std::isdigit(static_cast<unsigned char>(lower[i])))
++i;
tokens.push_back(lower.substr(start, i - start));
continue;
}
// Non-whitespace, non-alnum (punctuation, symbols)
{
size_t start = i;
while (i < lower.size() &&
!std::isspace(static_cast<unsigned char>(lower[i])) &&
!std::isalpha(static_cast<unsigned char>(lower[i])) &&
!std::isdigit(static_cast<unsigned char>(lower[i])))
++i;
if (i > start)
tokens.push_back(lower.substr(start, i - start));
}
}
return tokens;
}
// =========================================================================
// BPE algorithm
// =========================================================================
std::vector<std::string> ANSCLIPTokenizer::bpe(const std::string& word) const
{
// Check cache
auto cacheIt = m_cache.find(word);
if (cacheIt != m_cache.end())
return cacheIt->second;
// Byte-encode the word
std::vector<std::string> symbols;
for (unsigned char c : word)
symbols.push_back(m_byteEncoder[c]);
// Append </w> to the last symbol (CLIP end-of-word marker)
if (!symbols.empty())
symbols.back() += "</w>";
if (symbols.size() <= 1) {
m_cache[word] = symbols;
return symbols;
}
// Iteratively merge the highest-priority (lowest-rank) pair
while (symbols.size() > 1) {
// Find the pair with the lowest rank
int bestRank = INT_MAX;
int bestIdx = -1;
for (size_t j = 0; j + 1 < symbols.size(); ++j) {
auto it = m_bpeRanks.find({ symbols[j], symbols[j + 1] });
if (it != m_bpeRanks.end() && it->second < bestRank) {
bestRank = it->second;
bestIdx = static_cast<int>(j);
}
}
if (bestIdx < 0) break; // no mergeable pairs left
// Merge ALL occurrences of the best pair
std::string a = symbols[bestIdx];
std::string b = symbols[bestIdx + 1];
std::string merged = a + b;
std::vector<std::string> next;
size_t j = 0;
while (j < symbols.size()) {
if (j + 1 < symbols.size() && symbols[j] == a && symbols[j + 1] == b) {
next.push_back(merged);
j += 2;
} else {
next.push_back(symbols[j]);
j += 1;
}
}
symbols = std::move(next);
}
m_cache[word] = symbols;
return symbols;
}
// =========================================================================
// Load vocabulary from BPE merges file
// =========================================================================
bool ANSCLIPTokenizer::Load(const std::string& mergesFilePath)
{
initByteEncoder();
std::ifstream file(mergesFilePath);
if (!file.is_open())
return false;
std::string line;
std::vector<std::pair<std::string, std::string>> merges;
while (std::getline(file, line)) {
// Trim trailing \r \n
while (!line.empty() && (line.back() == '\r' || line.back() == '\n'))
line.pop_back();
// Skip empty lines and comments (#version: 0.2)
if (line.empty() || line[0] == '#')
continue;
size_t sp = line.find(' ');
if (sp == std::string::npos)
continue;
merges.emplace_back(line.substr(0, sp), line.substr(sp + 1));
}
file.close();
// Build BPE ranks
m_bpeRanks.clear();
for (int i = 0; i < static_cast<int>(merges.size()); ++i)
m_bpeRanks[merges[i]] = i;
// Build vocabulary (same order as OpenAI CLIP):
// [0..255] single byte-encoded chars
// [256..511] byte-encoded chars + </w>
// [512..512+N-1] merged tokens
// [49406] <|startoftext|>
// [49407] <|endoftext|>
m_encoder.clear();
int id = 0;
for (int b = 0; b < 256; ++b)
m_encoder[m_byteEncoder[b]] = id++;
for (int b = 0; b < 256; ++b)
m_encoder[m_byteEncoder[b] + "</w>"] = id++;
for (const auto& merge : merges)
m_encoder[merge.first + merge.second] = id++;
m_encoder["<|startoftext|>"] = id++;
m_encoder["<|endoftext|>"] = id++;
m_cache.clear();
m_loaded = true;
return true;
}
// =========================================================================
// Tokenize
// =========================================================================
TokenizerResult ANSCLIPTokenizer::Tokenize(const std::string& text, int maxLength) const
{
TokenizerResult result;
result.inputIds.resize(maxLength, 0); // pad with 0 (CLIP convention)
result.attentionMask.resize(maxLength, 0);
if (!m_loaded)
return result;
// BOS token
result.inputIds[0] = BOS_TOKEN;
result.attentionMask[0] = 1;
int pos = 1;
// Pre-tokenize -> BPE -> encode
for (const auto& word : preTokenize(text)) {
for (const auto& tok : bpe(word)) {
auto it = m_encoder.find(tok);
if (it != m_encoder.end() && pos < maxLength - 1) {
result.inputIds[pos] = it->second;
result.attentionMask[pos] = 1;
++pos;
}
}
}
// EOS token
if (pos < maxLength) {
result.inputIds[pos] = EOS_TOKEN;
result.attentionMask[pos] = 1;
}
return result;
}
}
+65
View File
@@ -0,0 +1,65 @@
#ifndef ANSCLIPTOKENIZER_H
#define ANSCLIPTOKENIZER_H
#pragma once
#include <string>
#include <vector>
#include <unordered_map>
#include <map>
#include <cstdint>
namespace ANSCENTER
{
struct TokenizerResult
{
std::vector<int64_t> inputIds;
std::vector<int64_t> attentionMask;
};
/// CLIP BPE tokenizer for text-prompted segmentation models.
///
/// Requires a BPE merges file (merges.txt from HuggingFace
/// openai/clip-vit-base-patch32). Place the file alongside the
/// ONNX model in the model folder.
class ANSCLIPTokenizer
{
public:
/// Load BPE vocabulary from a merges file.
/// @param mergesFilePath Path to the CLIP BPE merges file (merges.txt).
/// @return true on success.
bool Load(const std::string& mergesFilePath);
/// @return true if vocabulary has been loaded.
bool IsLoaded() const { return m_loaded; }
/// Tokenize a text prompt into input IDs and attention mask.
/// @param text The text to tokenize (e.g., "person").
/// @param maxLength Output sequence length (padded/truncated). Default 32.
/// @return TokenizerResult with inputIds and attentionMask vectors.
TokenizerResult Tokenize(const std::string& text, int maxLength = 32) const;
private:
bool m_loaded = false;
// Byte value (0-255) -> unicode string representation (CLIP byte encoding)
std::string m_byteEncoder[256];
// BPE merge: (token_a, token_b) -> priority rank (lower = merge first)
std::map<std::pair<std::string, std::string>, int> m_bpeRanks;
// Token string -> integer ID
std::unordered_map<std::string, int> m_encoder;
static constexpr int BOS_TOKEN = 49406; // <|startoftext|>
static constexpr int EOS_TOKEN = 49407; // <|endoftext|>
// BPE result cache (word -> BPE token list)
mutable std::unordered_map<std::string, std::vector<std::string>> m_cache;
void initByteEncoder();
std::vector<std::string> bpe(const std::string& word) const;
static std::vector<std::string> preTokenize(const std::string& text);
static std::string codepointToUtf8(int codepoint);
};
}
#endif
+398
View File
@@ -0,0 +1,398 @@
#include "ANSCUSTOMDetector.h"
#include "Utility.h"
#include <windows.h>
namespace ANSCENTER
{
std::string ANSCUSTOMDETECTOR::FindANSCustomLibraryName(const std::string& folderPath, const std::string& keyword) {
std::string searchPath = folderPath + "\\*.dll";
WIN32_FIND_DATAA findFileData;
HANDLE hFind = FindFirstFileA(searchPath.c_str(), &findFileData);
if (hFind == INVALID_HANDLE_VALUE) {
return ""; // Return an empty string if no file is found
}
do {
std::string fileName = findFileData.cFileName;
if (fileName.find(keyword) != std::string::npos) {
FindClose(hFind); // Close the handle before returning
return folderPath + "\\" + fileName; // Return the full path
}
} while (FindNextFileA(hFind, &findFileData) != 0);
FindClose(hFind); // Close the handle after finishing the loop
return ""; // Return an empty string if no match is found
}
bool ANSCUSTOMDETECTOR::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
optimizedModelFolder = _modelFolder;
return _customDetector->OptimizeModel(fp16);
}
bool ANSCUSTOMDETECTOR::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
std::string labelMap;
CreateCustomDetector();
//_customDetector->SetLoadEngineOnCreate(this->_loadEngineOnCreation);
return _customDetector->Initialize(_modelFolder,_modelConfig.detectionScoreThreshold, labelMap);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMDETECTOR::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
_modelFolder = modelFolder;
std::string labelMap;
_modelConfig = modelConfig;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
CreateCustomDetector();
_customDetector->SetLoadEngineOnCreate(this->_loadEngineOnCreation);
_isInitialized = _customDetector->Initialize(_modelFolder, _modelConfig.detectionScoreThreshold, labelMap);
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMDETECTOR::ConfigureParameters(Params& param) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
if (!_customDetector) {
this->_logger.LogError("ANSCUSTOMDETECTOR::ConfigureParamaters", "Custom detector is not initialized", __FILE__, __LINE__);
return false;
}
// Convert the vector of Params to a vector of CustomParams
CustomParams customParams;
_customDetector->ConfigureParameters(customParams);
// loop throught this->_params and set the values
param.ROI_Config.clear();
for (auto& cf : customParams.ROI_Config) {
ROIConfig ROIConfig;
ROIConfig.Rectangle = cf.Rectangle;
ROIConfig.Polygon = cf.Polygon;
ROIConfig.Line = cf.Line;
ROIConfig.MinItems = cf.MinItems;
ROIConfig.MaxItems = cf.MaxItems;
ROIConfig.Name = cf.Name;
ROIConfig.ROIMatch = cf.ROIMatch;
param.ROI_Config.push_back(ROIConfig);
}
param.ROI_Options.clear();
for (auto& op : customParams.ROI_Options) {
param.ROI_Options.push_back(op);
}
param.Parameters.clear();
for (auto& par : customParams.Parameters) {
Parameter Param;
Param.Name = par.Name;
Param.DataType = par.DataType;
Param.NoOfDecimals = par.NoOfDecimals;
Param.MaxValue = par.MaxValue;
Param.MinValue = par.MinValue;
Param.StartValue = par.StartValue;
Param.ListItems.clear();
for (auto& item : par.ListItems) {
Param.ListItems.push_back(item);
}
Param.DefaultValue = par.DefaultValue;
Param.Value = par.Value;
param.Parameters.push_back(Param);
}
param.ROI_Values.clear();
for (auto& roi : customParams.ROI_Values) {
ROIValue roiValue;
roiValue.ROIMatch = roi.ROIMatch;
roiValue.OriginalImageSize = roi.OriginalImageSize;
roiValue.Name = roi.Name;
roiValue.Option = roi.Option;
roiValue.ROIPoints.clear();
for (auto& point : roi.ROIPoints) {
Point customPoint;
customPoint.x = point.x;
customPoint.y = point.y;
roiValue.ROIPoints.push_back(customPoint);
}
param.ROI_Values.push_back(roiValue);
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::ConfigureParamaters", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMDETECTOR::SetParameters(const Params& param) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Convert the vector of Params to a vector of CustomParams
ANSODBase::SetParameters(param);
CustomParams customParams;
customParams.ROI_Config.clear();
for (auto& cf : param.ROI_Config) {
CustomROIConfig customROIConfig;
customROIConfig.Rectangle = cf.Rectangle;
customROIConfig.Polygon = cf.Polygon;
customROIConfig.Line = cf.Line;
customROIConfig.MinItems = cf.MinItems;
customROIConfig.MaxItems = cf.MaxItems;
customROIConfig.Name = cf.Name;
customROIConfig.ROIMatch = cf.ROIMatch;
customParams.ROI_Config.push_back(customROIConfig);
}
customParams.ROI_Options.clear();
for (auto& op : param.ROI_Options) {
customParams.ROI_Options.push_back(op);
}
customParams.Parameters.clear();
for (auto& par : param.Parameters) {
CustomParameter customParam;
customParam.Name = par.Name;
customParam.DataType = par.DataType;
customParam.NoOfDecimals = par.NoOfDecimals;
customParam.MaxValue = par.MaxValue;
customParam.MinValue = par.MinValue;
customParam.StartValue = par.StartValue;
customParam.ListItems.clear();
for (auto& item : par.ListItems) {
customParam.ListItems.push_back(item);
}
customParam.DefaultValue = par.DefaultValue;
customParam.Value = par.Value;
customParams.Parameters.push_back(customParam);
}
customParams.ROI_Values.clear();
for (auto& roi : param.ROI_Values) {
CustomROIValue roiValue;
roiValue.ROIMatch = roi.ROIMatch;
roiValue.OriginalImageSize = roi.OriginalImageSize;
roiValue.Name = roi.Name;
roiValue.Option = roi.Option;
roiValue.ROIPoints.clear();
for (auto& point : roi.ROIPoints) {
CustomPoint customPoint;
customPoint.x = point.x;
customPoint.y = point.y;
roiValue.ROIPoints.push_back(customPoint);
}
customParams.ROI_Values.push_back(roiValue);
}
return _customDetector->SetParameters(customParams);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::SetParamaters", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMDETECTOR::CreateCustomDetector() {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
Destroy();
//1. Load dynamic library
_modelFilePath = FindANSCustomLibraryName(_modelFolder, "ANSCustom");//
if (!FileExist(_modelFilePath)) {
_modelFilePath = CreateFilePath(_modelFolder, "ANSCustomCode.dll"); // This is the dynamic library
}
//2. Check if the file exists again
if (!FileExist(_modelFilePath)) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Initialize", "Cannot find ANSCustomCode.dll", __FILE__, __LINE__);
return false;
}
// 3. Load the library
std::wstring dllPath = String2WString(_modelFilePath);
hMod = LoadLibrary(dllPath.c_str());
if (!hMod) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Initialize", "Cannot open library", __FILE__, __LINE__);
return false;
}
CreatePluginFunc createPlugin = (CreatePluginFunc)GetProcAddress(hMod, "Create");
if (!createPlugin) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Initialize.", "Cannot load Create", __FILE__, __LINE__);
FreeLibrary(hMod);
return false;
}
_customDetector = std::unique_ptr<IANSCustomClass>(createPlugin());
if (!_customDetector) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Initialize", "Cannot create custom detector", __FILE__, __LINE__);
FreeLibrary(hMod);
return false;
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMDETECTOR::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
_modelConfig = modelConfig;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
labelMap.clear();
//1. Create custom detector
CreateCustomDetector();
//2. Load initial model
_customDetector->SetLoadEngineOnCreate(this->_loadEngineOnCreation);
_isInitialized = _customDetector->Initialize(_modelFolder, _modelConfig.detectionScoreThreshold,labelMap);
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSCUSTOMDETECTOR::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> ANSCUSTOMDETECTOR::RunInference(const cv::Mat& input,const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<Object> result;
try {
// Early returns - good practice maintained
if (!_licenseValid) {
this->_logger.LogError("ANSCUSTOMDETECTOR::RunInference",
"Invalid License", __FILE__, __LINE__);
return result;
}
if (!_isInitialized) {
this->_logger.LogError("ANSCUSTOMDETECTOR::RunInference",
"Model is not initialized", __FILE__, __LINE__);
return result;
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
return result;
}
// Run inference
std::vector<CustomObject> customResult = _customDetector->RunInference(input, camera_id);
// Pre-allocate to avoid reallocations
result.reserve(customResult.size());
//// Use emplace_back + move semantics for zero-copy construction
for (auto& customObj : customResult) {
Object obj;
obj.classId = customObj.classId;
obj.trackId = customObj.trackId;
obj.className = std::move(customObj.className);
obj.confidence = customObj.confidence;
obj.mask = std::move(customObj.mask);
obj.box = customObj.box;
obj.extraInfo = std::move(customObj.extraInfo);
obj.kps = std::move(customObj.kps);
obj.polygon = std::move(customObj.polygon);
obj.cameraId = std::move(customObj.cameraId);
result.emplace_back(std::move(obj));
}
if (_trackerEnabled) {
result = ApplyTracking(result, camera_id);
if (_stabilizationEnabled) result = StabilizeDetections(result, camera_id);
}
return result; // RVO applies, no copy
}
catch (const std::exception& e) { // Catch by const reference
this->_logger.LogFatal("ANSCUSTOMDETECTOR::RunInference",
e.what(), __FILE__, __LINE__);
return {}; // Return empty vector
}
}
ANSCUSTOMDETECTOR::~ANSCUSTOMDETECTOR() {
try {
//this->_logger.LogDebug("ANSCUSTOMDETECTOR::~ANSCUSTOMDETECTOR()", "Release ANSCUSTOMDETECTOR ", __FILE__, __LINE__);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::~ANSCUSTOMDETECTOR()", e.what(), __FILE__, __LINE__);
}
}
bool ANSCUSTOMDETECTOR::Destroy() {
try {
if (_customDetector) {
_customDetector->Destroy();
_customDetector.reset();
}
if (hMod) {
FreeLibrary(hMod);
hMod = nullptr;
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSCUSTOMDETECTOR::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
}
//std::vector<Object> ANSCUSTOMDETECTOR::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> result;
// try {
// if (!_licenseValid) {
// this->_logger.LogError("ANSCUSTOMDETECTOR::RunInference", "Invalid License", __FILE__, __LINE__);
// return result;
// }
// if (!_isInitialized) {
// this->_logger.LogError("ANSCUSTOMDETECTOR::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return result;
// }
// // Run inference
// if (input.empty()) return result;
// if ((input.cols < 10) || (input.rows < 10)) return result;
// std::vector<CustomObject> customResult;
// customResult = _customDetector->RunInference(input,camera_id);
// for (int i = 0; i < customResult.size(); i++) {
// Object obj;
// obj.classId = customResult[i].classId;
// obj.trackId = customResult[i].trackId;
// obj.className = customResult[i].className;
// obj.confidence = customResult[i].confidence;
// obj.mask = customResult[i].mask;
// obj.box = customResult[i].box;
// obj.extraInfo = customResult[i].extraInfo;
// obj.kps = customResult[i].kps;
// obj.polygon = customResult[i].polygon;
// obj.cameraId = customResult[i].cameraId;
// //obj.attributes = customResult[i].attributes;
// result.push_back(obj);
// }
// return result;
// }
// catch (std::exception& e) {
// result.clear();
// this->_logger.LogFatal("ANSCUSTOMDETECTOR::RunInference", e.what(), __FILE__, __LINE__);
// return result;
// }
//}
+30
View File
@@ -0,0 +1,30 @@
#ifndef ANSCUSTOMDETECTOR_H
#define ANSCUSTOMDETECTOR_H
#pragma once
#include "ANSCustomCode.h"
#include "ANSEngineCommon.h"
typedef IANSCustomClass* (*CreatePluginFunc)();
namespace ANSCENTER {
class ANSENGINE_API ANSCUSTOMDETECTOR :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
virtual bool ConfigureParameters(Params& param) override;
virtual bool SetParameters(const Params& param)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSCUSTOMDETECTOR();
private:
std::string _modelFilePath;
std::string _modelCfgPath;
std::unique_ptr<IANSCustomClass> _customDetector;
HMODULE hMod= nullptr;
std::string FindANSCustomLibraryName(const std::string& folderPath, const std::string& keyword);
bool CreateCustomDetector();
};
}
#endif
+757
View File
@@ -0,0 +1,757 @@
#include "ANSCUSTOMPY.h"
#include <filesystem>
#include <json.hpp>
using json = nlohmann::json;
namespace fs = std::filesystem;
namespace py = pybind11;
namespace {
using ThreadLocalModelMap = std::unordered_map<int, std::shared_ptr<ANSCENTER::ModelWrapper>>;
ThreadLocalModelMap& GetThreadLocalState() {
thread_local ThreadLocalModelMap threadState;
return threadState;
}
}
std::string CustomParametersToJson(const ANSCENTER::Params& params) {
json root;
// ROI_Config
for (const auto& cfg : params.ROI_Config) {
json j;
j["Rectangle"] = cfg.Rectangle;
j["Polygon"] = cfg.Polygon;
j["Line"] = cfg.Line;
j["MinItems"] = cfg.MinItems;
j["MaxItems"] = cfg.MaxItems;
j["Name"] = cfg.Name;
j["ROI-Match"] = cfg.ROIMatch;
root["ROI_Config"].push_back(j);
}
// ROI_Options
for (const auto& opt : params.ROI_Options) {
root["ROI_Options"].push_back(opt);
}
// Parameters
for (const auto& param : params.Parameters) {
json j;
j["Name"] = param.Name;
j["DataType"] = param.DataType;
j["NoOfdecimals"] = param.NoOfDecimals;
j["MaxValue"] = param.MaxValue;
j["MinValue"] = param.MinValue;
j["StartValue"] = param.StartValue;
j["ListItems"] = param.ListItems;
j["DefaultValue"] = param.DefaultValue;
j["Value"] = param.Value;
root["Parameters"].push_back(j);
}
// ROI_Values
for (const auto& rv : params.ROI_Values) {
json j;
j["ROI-Match"] = rv.ROIMatch;
for (const auto& pt : rv.ROIPoints) {
j["ROIPoints"].push_back({ {"x", pt.x}, {"y", pt.y} });
}
j["Option"] = rv.Option;
j["Name"] = rv.Name;
j["OriginalImageSize"] = rv.OriginalImageSize;
root["ROI_Values"].push_back(j);
}
// Convert to compact JSON string (no whitespace)
std::string jsonString = root.dump(); // dump() without indent = compact
return jsonString;
}
namespace ANSCENTER
{
std::once_flag PythonRuntime::initFlag;
std::unique_ptr<PythonRuntime> PythonRuntime::instance = nullptr;
std::atomic<bool> PythonRuntime::_isInitialized{ false };
std::atomic<bool> PythonRuntime::shuttingDown{ false };
std::mutex PythonRuntime::_instanceMutex;
std::mutex ANSCUSTOMPY::_globalInstancesMutex;
std::unordered_set<ANSCUSTOMPY*> ANSCUSTOMPY::_globalInstances;
std::atomic<int> ANSCUSTOMPY::globalInstanceCounter{ 1 };
// Declare GetPythonRuntime as a friend function of PythonRuntime
PythonRuntime& GetPythonRuntime(const std::wstring& pythonHome);
PythonRuntime& GetPythonRuntime(const std::wstring& pythonHome) {
std::call_once(PythonRuntime::initFlag, [&]() {
std::lock_guard<std::mutex> lock(PythonRuntime::_instanceMutex);
PythonRuntime::instance = std::unique_ptr<PythonRuntime>(new PythonRuntime(pythonHome));
});
return *PythonRuntime::instance;
}
PythonRuntime::PythonRuntime(const std::wstring& home) : pythonHome(home) {
InitializeInterpreter();
}
bool PythonRuntime::IsInitialized() {
return _isInitialized.load(std::memory_order_acquire) && Py_IsInitialized();
}
void PythonRuntime::InitializeInterpreter() {
if (!pythonHome.empty()) {
#pragma warning(push)
#pragma warning(disable : 4996)
Py_SetPythonHome(pythonHome.c_str());
#pragma warning(pop)
}
py::initialize_interpreter();
_isInitialized.store(true, std::memory_order_release);
PatchPythonStreamsSafe();
}
void PythonRuntime::PatchPythonStreamsSafe() {
try {
py::exec(R"(
import sys, io
class DummyStream(io.StringIO):
def write(self, txt): pass
def flush(self): pass
if sys.stdout is None or not hasattr(sys.stdout, "write"):
sys.stdout = DummyStream()
if sys.stderr is None or not hasattr(sys.stderr, "write"):
sys.stderr = DummyStream()
)");
}
catch (const py::error_already_set& e) {
std::cerr << "[PatchPythonStreamsSafe] Python error: " << e.what() << std::endl;
}
}
void PythonRuntime::AddToSysPath(const std::string& path) {
if (!IsInitialized()) {
std::cerr << "[PythonRuntime::AddToSysPath] Interpreter not initialized!\n";
return;
}
py::gil_scoped_acquire gil;
py::module_ sys = py::module_::import("sys");
sys.attr("path").attr("insert")(0, path);
}
void PythonRuntime::Shutdown() {
std::lock_guard<std::mutex> lock(_instanceMutex);
if (shuttingDown.exchange(true)) return;
if (_isInitialized.load(std::memory_order_acquire)) {
try {
py::gil_scoped_acquire gil;
py::module::import("gc").attr("collect")();
}
catch (...) {
std::cerr << "[PythonRuntime::Shutdown] GC failed (ignored)." << std::endl;
}
try {
py::finalize_interpreter();
}
catch (...) {
std::cerr << "[PythonRuntime::Shutdown] finalize_interpreter failed." << std::endl;
}
_isInitialized.store(false, std::memory_order_release);
}
instance.reset();
}
/// <summary>
/// /// Initializes the ANSCUSTOMPY model with the given parameters.
/// </summary>
ANSCUSTOMPY::ANSCUSTOMPY() {
_instanceId = globalInstanceCounter.fetch_add(1, std::memory_order_relaxed);
_destroyed = false;
_isInitialized = false;
}
ANSCUSTOMPY::~ANSCUSTOMPY() {
try {
if (PythonRuntime::IsInitialized())
Destroy();
}
catch (...) {
std::cerr << "[~ANSCUSTOMPY] Exception during destruction." << std::endl;
}
}
void ANSCUSTOMPY::RegisterSelf() {
{
std::lock_guard<std::mutex> lock(_globalInstancesMutex);
_globalInstances.insert(this);
}
auto& threadState = GetThreadLocalState();
if (!_wrapper || !_wrapper->modelFactory) return;
if (threadState.find(_instanceId) == threadState.end()) {
if (_wrapper && _wrapper->modelFactory) {
py::gil_scoped_acquire gil;
auto newWrapper = std::make_shared<ModelWrapper>();
newWrapper->modelFactory = _wrapper->modelFactory;
newWrapper->map[0] = std::make_shared<py::object>(_wrapper->modelFactory());
threadState[_instanceId] = newWrapper;
}
}
}
void ANSCUSTOMPY::UnregisterSelf() {
std::lock_guard<std::mutex> lock(_globalInstancesMutex);
_globalInstances.erase(this);
}
void ANSCUSTOMPY::ClearThreadState() {
auto& threadState = GetThreadLocalState();
auto it = threadState.find(_instanceId);
if (it != threadState.end()) {
if (PythonRuntime::IsInitialized()) {
py::gil_scoped_acquire gil;
it->second->map.clear();
}
else {
it->second->map.clear();
}
threadState.erase(it);
}
}
bool ANSCUSTOMPY::EnsureModelReady(const std::string& methodName) {
if (!_isInitialized || _destroyed || !PythonRuntime::IsInitialized()) {
return false;
}
//auto& threadState = GetThreadLocalState();
//if (threadState.find(_instanceId) == threadState.end()) {
// if (_wrapper && _wrapper->modelFactory) {
// py::gil_scoped_acquire gil;
// auto newWrapper = std::make_shared<ModelWrapper>();
// newWrapper->modelFactory = _wrapper->modelFactory;
// newWrapper->map[0] = std::make_shared<py::object>(_wrapper->modelFactory());
// threadState[_instanceId] = newWrapper;
// }
// else {
// return false;
// }
//}
return true;
}
py::object ANSCUSTOMPY::GetThreadLocalModel() {
auto& threadState = GetThreadLocalState();
auto it = threadState.find(_instanceId);
if (_destroyed || it == threadState.end() || !it->second) {
return py::none();
}
auto& modelMap = it->second->map;
auto modelIt = modelMap.find(0);
if (modelIt == modelMap.end()) {
py::gil_scoped_acquire gil;
auto obj = it->second->modelFactory();
if (obj.is_none()) return py::none();
auto wrapped = std::make_shared<py::object>(obj);
modelMap[0] = wrapped;
return *wrapped;
}
return *modelIt->second;
}
bool ANSCUSTOMPY::Destroy() {
//std::lock_guard<std::recursive_mutex> lock(_mutex);
if (_destroyed) return true;
_destroyed = true;
ClearThreadState();
UnregisterSelf();
return true;
}
void ANSCUSTOMPY::SafeShutdownAll(bool waitForShutdown) {
std::unordered_set<ANSCUSTOMPY*> instancesCopy;
{
std::lock_guard<std::mutex> lock(_globalInstancesMutex);
instancesCopy = _globalInstances;
}
for (ANSCUSTOMPY* instance : instancesCopy) {
if (instance) {
try {
instance->ClearThreadState();
instance->Destroy();
}
catch (...) {
std::cerr << "[SafeShutdownAll] Destroy failed." << std::endl;
}
}
}
std::this_thread::sleep_for(std::chrono::milliseconds(300));
auto shutdownFunc = []() {
try {
std::this_thread::sleep_for(std::chrono::milliseconds(300));
int retry = 0;
while (!ANSCUSTOMPY::_globalInstances.empty() && retry++ < 100) {
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
if (PythonRuntime::IsInitialized()) {
py::gil_scoped_acquire gil;
py::module::import("gc").attr("collect")();
PythonRuntime::Shutdown();
}
}
catch (...) {
std::cerr << "[SafeShutdownAll] Shutdown failed." << std::endl;
}
};
if (waitForShutdown) shutdownFunc();
else std::thread(shutdownFunc).detach();
}
bool ANSCUSTOMPY::InitializePythonModel(const std::string& moduleName, const std::string& fullPath) {
if (!PythonRuntime::IsInitialized()) {
_logger.LogError("[InitializePythonModel] Python error:\n", "Python interpreter is not initialized.", __FILE__, __LINE__);
return false;
}
py::gil_scoped_acquire gil;
try {
py::exec("import logging; logging.getLogger('ultralytics').setLevel(logging.CRITICAL)", py::globals());
py::dict locals;
std::ostringstream code;
code << "import importlib.util\n"
<< "spec = importlib.util.spec_from_file_location('" << moduleName << "', r'" << fullPath << "')\n"
<< "mod = importlib.util.module_from_spec(spec)\n"
<< "spec.loader.exec_module(mod)\n";
py::exec(code.str(), py::globals(), locals);
py::object mod = locals["mod"];
py::object modelClass = mod.attr("ANSModel");
auto& threadState = GetThreadLocalState();
auto& wrapper = threadState[_instanceId]; // ensure per-thread-per-instance
if (!wrapper) wrapper = std::make_shared<ModelWrapper>();
wrapper->modelFactory = [modelClass]() {
py::gil_scoped_acquire gil;
py::object instance = modelClass();
return instance;};
_wrapper = wrapper; // Store for main thread access too
return true;
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("InitializePythonModel", std::string("Python error: ") + e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMPY::Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath, const std::string& modelZipPassword,
std::string& labelMap) {
//std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Step 1: Base model init
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
_modelConfig = modelConfig;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2) _modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2) _modelConfig.modelConfThreshold = 0.5;
// Step 2: Validate model folder
if (_modelFolder.empty()) {
_logger.LogError("ANSCUSTOMPY::Initialize", "_modelFolder is not set.", __FILE__, __LINE__);
return false;
}
// Step 3: Set Python Home
std::wstring pythonHome;
const fs::path sharedPython = "C:/ProgramData/ANSCENTER/Python311";
const fs::path sharedDll = sharedPython / "python311.dll";
if (fs::exists(sharedDll)) {
pythonHome = sharedPython.wstring();
}
else {
pythonHome = (fs::path(_modelFolder) / "python_env").wstring();
}
// Step 4: Initialize Python and sys.path
auto& runtime = GetPythonRuntime(pythonHome);
runtime.AddToSysPath(_modelFolder);
// Step 5: Load Python model class from unique module
//_scriptPath = _modelFolder + "/ansinfer.py";
_scriptPath = fs::path(_modelFolder).append("ansinfer.py").string();
_moduleName = "ansinfer_" + std::to_string(_instanceId);
if (!InitializePythonModel(_moduleName, _scriptPath)) {
_logger.LogError("ANSCUSTOMPY::Initialize", "Failed to load Python model.", __FILE__, __LINE__);
return false;
}
// Step 6: Load class names
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName) {
_logger.LogDebug("ANSCUSTOMPY::LoadModel. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
_logger.LogDebug("ANSCUSTOMPY::LoadModel. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
_destroyed = false;
_isInitialized = true;
RegisterSelf();
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMPY::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
//std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Step 0: Assign unique instance ID
if (!ANSODBase::LoadModel(modelZipFilePath, modelZipPassword))
return false;
if (_modelFolder.empty()) {
_logger.LogError("ANSCUSTOMPY::LoadModel", "_modelFolder is not set.", __FILE__, __LINE__);
return false;
}
// Step 3: Set Python home
std::wstring pythonHome;
const fs::path sharedPython = "C:/ProgramData/ANSCENTER/Python311";
const fs::path sharedDll = sharedPython / "python311.dll";
if (fs::exists(sharedDll)) {
pythonHome = sharedPython.wstring();
}
else {
pythonHome = (fs::path(_modelFolder) / "python_env").wstring();
}
// Step 4: Initialize Python and sys.path
auto& runtime = GetPythonRuntime(pythonHome);
runtime.AddToSysPath(_modelFolder);
// Step 5: Load Python model class from unique module
//_scriptPath = _modelFolder + "/ansinfer.py";
_scriptPath = fs::path(_modelFolder).append("ansinfer.py").string();
_moduleName = "ansinfer_" + std::to_string(_instanceId);
if (!InitializePythonModel(_moduleName, _scriptPath)) {
_logger.LogError("ANSCUSTOMPY::LoadModel", "Failed to load Python model.", __FILE__, __LINE__);
return false;
}
_destroyed = false;
_isInitialized = true;
RegisterSelf();
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMPY::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap)
{
//std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Step 0: Assign unique instance ID early
if (!ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap))
return false;
_modelConfig = modelConfig;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2) _modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2) _modelConfig.modelConfThreshold = 0.5;
// Step 2: Validate model folder
if (_modelFolder.empty()) {
_logger.LogError("ANSCUSTOMPY::Initialize", "_modelFolder is not set.", __FILE__, __LINE__);
return false;
}
// Step 3: Set Python home
std::wstring pythonHome;
const fs::path sharedPython = "C:/ProgramData/ANSCENTER/Python311";
const fs::path sharedDll = sharedPython / "python311.dll";
if (fs::exists(sharedDll)) {
pythonHome = sharedPython.wstring();
}
else {
pythonHome = (fs::path(_modelFolder) / "python_env").wstring();
}
// Step 4: Initialize Python and sys.path
auto& runtime = GetPythonRuntime(pythonHome);
runtime.AddToSysPath(_modelFolder);
// Step 5: Load Python model class from unique module
//_scriptPath = _modelFolder + "/ansinfer.py";
_scriptPath = fs::path(_modelFolder).append("ansinfer.py").string();
_moduleName = "ansinfer_" + std::to_string(_instanceId);
if (!InitializePythonModel(_moduleName, _scriptPath)) {
_logger.LogError("ANSCUSTOMPY::Initialize", "Failed to load Python model.", __FILE__, __LINE__);
return false;
}
// Step 6: Load class labels
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName) {
_logger.LogDebug("ANSCUSTOMPY::LoadModel", "Load classes from string", __FILE__, __LINE__);
LoadClassesFromString();
}
else {
_logger.LogDebug("ANSCUSTOMPY::LoadModel", "Load classes from file", __FILE__, __LINE__);
LoadClassesFromFile();
}
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
_destroyed = false;
_isInitialized = true;
RegisterSelf();
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSCUSTOMPY::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomPyCam");
}
std::vector<Object> ANSCUSTOMPY::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::vector<Object> results;
if (!_isInitialized) {
_logger.LogError("ANSCUSTOMPY::RunInference", "Model is not initialized.", __FILE__, __LINE__);
return results;
}
if (!_licenseValid) {
_logger.LogError("ANSCUSTOMPY::RunInference", "Invalid License", __FILE__, __LINE__);
return results;
}
if (input.empty()) {
_logger.LogError("ANSCUSTOMPY::RunInference", "Input image is empty.", __FILE__, __LINE__);
return results;
}
try {
std::string jsonStr = RunPythonInferenceFromMat(input);
if (!jsonStr.empty()) {
results = ANSUtilityHelper::GetDetectionsFromString(jsonStr);
}
else {
_logger.LogWarn("ANSCUSTOMPY::RunInference", "Empty JSON result from Python inference.", __FILE__, __LINE__);
}
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::RunInference", e.what(), __FILE__, __LINE__);
}
return results;
}
std::string ANSCUSTOMPY::RunPythonInferenceFromMat(const cv::Mat& image) {
if (!EnsureModelReady("RunPythonInferenceFromMat"))
return "";
if (image.empty())
return "";
try {
py::object model = GetThreadLocalModel();
if (model.is_none()) {
_logger.LogError("ANSCUSTOMPY::RunPythonInferenceFromMat", "Model is None.", __FILE__, __LINE__);
return "";
}
py::gil_scoped_acquire gil;
if (!py::hasattr(model, "run_inference")) {
_logger.LogError("ANSCUSTOMPY::RunPythonInferenceFromMat", "Model has no 'run_inference' method.", __FILE__, __LINE__);
return "";
}
py::bytes image_bytes(reinterpret_cast<const char*>(image.data), image.total() * image.elemSize());
py::object result = model.attr("run_inference")(image_bytes, image.cols, image.rows, image.channels());
return result.cast<std::string>();
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("RunPythonInferenceFromMat", e.what(), __FILE__, __LINE__);
return "";
}
catch (const std::exception& e) {
_logger.LogFatal("RunPythonInferenceFromMat", e.what(), __FILE__, __LINE__);
return "";
}
}
bool ANSCUSTOMPY::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (!EnsureModelReady("OptimizeModel")) return false;
try {
py::object model = GetThreadLocalModel();
if (model.is_none()) {
_logger.LogError("ANSCUSTOMPY::OptimizeModel", "Model is None.", __FILE__, __LINE__);
return false;
}
py::gil_scoped_acquire gil;
if (!py::hasattr(model, "model_optimize")) {
_logger.LogError("ANSCUSTOMPY::OptimizeModel", "Python model has no method 'model_optimize'.", __FILE__, __LINE__);
return false;
}
py::object result;
try {
result = model.attr("model_optimize")(fp16);
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("ANSCUSTOMPY::OptimizeModel", std::string("Exception in 'model_optimize': ") + e.what(), __FILE__, __LINE__);
return false;
}
if (result.is_none()) {
_logger.LogError("ANSCUSTOMPY::OptimizeModel", "Python 'model_optimize' returned None.", __FILE__, __LINE__);
return false;
}
if (!py::isinstance<py::bool_>(result)) {
_logger.LogError("ANSCUSTOMPY::OptimizeModel", "Unexpected return type from 'model_optimize'. Expected bool.", __FILE__, __LINE__);
return false;
}
bool success = result.cast<bool>();
if (success) {
optimizedModelFolder = fs::path(_scriptPath).parent_path().string();
}
return success;
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("ANSCUSTOMPY::OptimizeModel", e.what(), __FILE__, __LINE__);
return false;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::OptimizeModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMPY::ConfigureParameters(Params& param) {
if (!EnsureModelReady("ConfigureParamaters")) return false;
try {
py::object model = GetThreadLocalModel();
if (model.is_none()) {
_logger.LogError("ANSCUSTOMPY::ConfigureParamaters", "Model is None.", __FILE__, __LINE__);
return false;
}
py::gil_scoped_acquire gil;
if (!py::hasattr(model, "configureParameters")) {
_logger.LogError("ANSCUSTOMPY::ConfigureParamaters", "Python model missing 'configureParameters' method.", __FILE__, __LINE__);
return false;
}
py::object result;
try {
result = model.attr("configureParameters")();
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("ANSCUSTOMPY::ConfigureParamaters", std::string("Exception in 'configureParameters': ") + e.what(), __FILE__, __LINE__);
return false;
}
if (result.is_none()) {
_logger.LogError("ANSCUSTOMPY::ConfigureParamaters", "Returned None from Python method.", __FILE__, __LINE__);
return false;
}
if (!py::isinstance<py::str>(result)) {
_logger.LogError("ANSCUSTOMPY::ConfigureParamaters", "Expected Python string return from 'configureParameters'.", __FILE__, __LINE__);
return false;
}
std::string jsonStr = result.cast<std::string>();
if (jsonStr.empty()) {
_logger.LogError("ANSCUSTOMPY::ConfigureParamaters", "Empty JSON string returned from Python.", __FILE__, __LINE__);
return false;
}
param = ANSUtilityHelper::ParseCustomParameters(jsonStr);
return true;
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("ANSCUSTOMPY::ConfigureParamaters", e.what(), __FILE__, __LINE__);
return false;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::ConfigureParamaters", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSCUSTOMPY::SetParameters(const Params& param) {
if (!EnsureModelReady("SetParamaters")) return false;
try {
std::string jsonStr = CustomParametersToJson(param);
if (jsonStr.empty()) {
_logger.LogError("ANSCUSTOMPY::SetParamaters", "Serialized parameter string is empty.", __FILE__, __LINE__);
return false;
}
py::object model = GetThreadLocalModel();
if (model.is_none()) {
_logger.LogError("ANSCUSTOMPY::SetParamaters", "Model is None.", __FILE__, __LINE__);
return false;
}
py::gil_scoped_acquire gil;
if (!py::hasattr(model, "setParameters")) {
_logger.LogError("ANSCUSTOMPY::SetParamaters", "Python model has no method 'setParameters'.", __FILE__, __LINE__);
return false;
}
py::object result;
try {
result = model.attr("setParameters")(jsonStr);
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("ANSCUSTOMPY::SetParamaters", std::string("Exception in 'setParameters': ") + e.what(), __FILE__, __LINE__);
return false;
}
if (result.is_none()) {
_logger.LogError("ANSCUSTOMPY::SetParamaters", "Python method returned None.", __FILE__, __LINE__);
return false;
}
if (!py::isinstance<py::bool_>(result)) {
_logger.LogError("ANSCUSTOMPY::SetParamaters", "Python method returned non-boolean type.", __FILE__, __LINE__);
return false;
}
return result.cast<bool>();
}
catch (const py::error_already_set& e) {
PyErr_Print();
_logger.LogFatal("ANSCUSTOMPY::SetParamaters", std::string("Python error: ") + e.what(), __FILE__, __LINE__);
return false;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSCUSTOMPY::SetParamaters", e.what(), __FILE__, __LINE__);
return false;
}
}
}
+340
View File
@@ -0,0 +1,340 @@
#ifndef ANSCUSTOMPY_H
#define ANSCUSTOMPY_H
#pragma once
#include "ANSEngineCommon.h"
#include <iostream>
#include <iomanip>
#include <string>
#include <vector>
#include <queue>
#include <fstream>
#include <thread>
#include <future>
#include <atomic>
#include <mutex>
#include <cmath>
#include <opencv2/opencv.hpp>
#include <pybind11/embed.h>
namespace py = pybind11;
namespace ANSCENTER {
struct ModelWrapper {
std::unordered_map<int, std::shared_ptr<pybind11::object>> map;
std::function<pybind11::object()> modelFactory;
};
class PythonRuntime {
public:
static bool IsInitialized();
static void Shutdown();
void AddToSysPath(const std::string& path);
friend PythonRuntime& GetPythonRuntime(const std::wstring& pythonHome);
private:
explicit PythonRuntime(const std::wstring& home);
void InitializeInterpreter();
void PatchPythonStreamsSafe();
static std::once_flag initFlag;
static std::unique_ptr<PythonRuntime> instance;
static std::atomic<bool> _isInitialized;
static std::atomic<bool> shuttingDown;
static std::mutex _instanceMutex;
std::wstring pythonHome;
};
PythonRuntime& GetPythonRuntime(const std::wstring& pythonHome);
class ANSENGINE_API ANSCUSTOMPY : public ANSODBase {
public:
ANSCUSTOMPY();
~ANSCUSTOMPY();
bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath,
const std::string& modelZipPassword, std::string& labelMap) override;
bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName,
std::string className, const std::string& modelFolder, std::string& labelMap) override;
bool ConfigureParameters(Params& param) override;
bool SetParameters(const Params& param) override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input) override;
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id) override;
bool Destroy() override;
static void SafeShutdownAll(bool waitForShutdown);
private:
bool _destroyed = false;
bool _isInitialized = false;
std::recursive_mutex _mutex;
pybind11::object pyModelClass;
std::shared_ptr<ModelWrapper> _wrapper;
int _instanceId;
std::string _scriptPath;
std::string _moduleName;
bool InitializePythonModel(const std::string& moduleName, const std::string& fullPath);
pybind11::object GetThreadLocalModel();
bool EnsureModelReady(const std::string& methodName);
void ClearThreadState();
void RegisterSelf();
void UnregisterSelf();
std::string RunPythonInferenceFromMat(const cv::Mat& image);
static std::mutex _globalInstancesMutex;
static std::unordered_set<ANSCUSTOMPY*> _globalInstances;
static std::atomic<int> globalInstanceCounter;
};
} // namespace ANSCENTER
#endif // ANSCUSTOMPY_H
//import numpy as np
//import os
//import json
//from ultralytics import YOLO
//from typing import List
//
//class ROIPoint :
// def __init__(self, x, y) :
// self.x = x
// self.y = y
//
// def to_dict(self) :
// return { "x": self.x, "y" : self.y }
//
// class ROIConfig :
// def __init__(self, rectangle, polygon, line, min_items, max_items, name, roi_match) :
// self.Rectangle = rectangle
// self.Polygon = polygon
// self.Line = line
// self.MinItems = min_items
// self.MaxItems = max_items
// self.Name = name
// self.ROI_Match = roi_match
//
// def to_dict(self) :
// return {
// "Rectangle": self.Rectangle,
// "Polygon" : self.Polygon,
// "Line" : self.Line,
// "MinItems" : self.MinItems,
// "MaxItems" : self.MaxItems,
// "Name" : self.Name,
// "ROI-Match" : self.ROI_Match
//}
//
//class Parameter :
// def __init__(self, name, datatype, no_of_decimals, max_val, min_val, start_val, list_items, default_val, value) :
// self.Name = name
// self.DataType = datatype
// self.NoOfdecimals = no_of_decimals
// self.MaxValue = max_val
// self.MinValue = min_val
// self.StartValue = start_val
// self.ListItems = list_items
// self.DefaultValue = default_val
// self.Value = value
//
// def to_dict(self) :
// return {
// "Name": self.Name,
// "DataType" : self.DataType,
// "NoOfdecimals" : self.NoOfdecimals,
// "MaxValue" : self.MaxValue,
// "MinValue" : self.MinValue,
// "StartValue" : self.StartValue,
// "ListItems" : self.ListItems,
// "DefaultValue" : self.DefaultValue,
// "Value" : self.Value
//}
//
//class ROIValue :
// def __init__(self, roi_match, roi_points, option, name, original_image_size) :
// self.ROI_Match = roi_match
// self.ROIPoints = roi_points # list of ROIPoint
// self.Option = option
// self.Name = name
// self.OriginalImageSize = original_image_size
//
// def to_dict(self) :
// return {
// "ROI-Match": self.ROI_Match,
// "ROIPoints" : [p.to_dict() for p in self.ROIPoints] ,
// "Option" : self.Option,
// "Name" : self.Name,
// "OriginalImageSize" : self.OriginalImageSize
//}
//
//class Params :
// def __init__(self) :
// self.ROI_Config : List[ROIConfig] = []
// self.ROI_Options : List[str] = []
// self.Parameters : List[Parameter] = []
// self.ROI_Values : List[ROIValue] = []
//
// def to_dict(self) :
// return {
// "ROI_Config": [cfg.to_dict() for cfg in self.ROI_Config] ,
// "ROI_Options" : self.ROI_Options,
// "Parameters" : [p.to_dict() for p in self.Parameters] ,
// "ROI_Values" : [v.to_dict() for v in self.ROI_Values]
//}
//
//def to_json(self) :
// return json.dumps(self.to_dict(), indent = 2)
//
// class ANSModel :
// def __init__(self, model_name = "yolo11n.pt") :
// current_dir = os.path.dirname(os.path.abspath(__file__))
// model_path = os.path.join(current_dir, model_name)
// self.model = YOLO(model_path)
//
// # Define empty but structured parameter dictionary
// self.params = {
// "ROI_Config": [] ,
// "ROI_Options" : [] ,
// "Parameters" : [] ,
// "ROI_Values" : []
//}
//
//def run_inference(self, image_bytes: bytes, width : int, height : int, channels : int)->str:
//np_arr = np.frombuffer(image_bytes, dtype = np.uint8)
//image = np_arr.reshape((height, width, channels))
//
//results = self.model(image)
//output = { "results": [] }
//
//for result in results :
//for box in result.boxes :
// cls_id = int(box.cls[0])
// conf = float(box.conf[0])
// class_name = self.model.names[cls_id]
// x1, y1, x2, y2 = box.xyxy[0].tolist()
// x = int(x1)
// y = int(y1)
// w = int(x2 - x1)
// h = int(y2 - y1)
//
// output["results"].append({
// "class_id": cls_id,
// "track_id" : -1,
// "class_name" : class_name,
// "prob" : conf,
// "x" : x,
// "y" : y,
// "width" : w,
// "height" : h,
// "mask" : "",
// "extra_info" : "",
// "camera_id" : "",
// "polygon" : "",
// "kps" : ""
// })
//
// return json.dumps(output)
//
// def model_optimize(self, fp16: bool) -> bool:
//try :
// print(f"[model_optimize] Called with fp16={fp16}")
// # self.model.export(...) # real optimization here
// return True
// except Exception as e :
//print(f"[model_optimize] Error: {e}")
//return False
//
//def configureParameters(self)->str :
//# --- Add ROI_Config entries ---
// self.params["ROI_Config"].append({
// "Rectangle": True,
// "Polygon" : True,
// "Line" : False,
// "MinItems" : 0,
// "MaxItems" : 3,
// "Name" : "Traffic Light",
// "ROI-Match" : "All Corners"
// })
//
// self.params["ROI_Config"].append({
// "Rectangle": True,
// "Polygon" : False,
// "Line" : False,
// "MinItems" : 1,
// "MaxItems" : 1,
// "Name" : "Car Zone",
// "ROI-Match" : "All Corners"
// })
//
// self.params["ROI_Config"].append({
// "Rectangle": False,
// "Polygon" : False,
// "Line" : True,
// "MinItems" : 1,
// "MaxItems" : 2,
// "Name" : "Cross Line",
// "ROI-Match" : "All Corners"
// })
//
//# --- Add ROI_Options entries ---
// self.params["ROI_Options"] += ["Inside ROI", "Inside ROI", "Both Directions"]
//
//# --- Add Parameters entries ---
// self.params["Parameters"].append({
// "Name": "Para1",
// "DataType" : "Boolean",
// "NoOfdecimals" : 0,
// "MaxValue" : 0,
// "MinValue" : 0,
// "StartValue" : "",
// "ListItems" : [] ,
// "DefaultValue" : "",
// "Value" : "true"
// })
//
// self.params["Parameters"].append({
// "Name": "Para2",
// "DataType" : "Integer",
// "NoOfdecimals" : 0,
// "MaxValue" : 5,
// "MinValue" : 1,
// "StartValue" : "2",
// "ListItems" : [] ,
// "DefaultValue" : "",
// "Value" : "3"
// })
//
// self.params["Parameters"].append({
// "Name": "Para3",
// "DataType" : "List-Single",
// "NoOfdecimals" : 0,
// "MaxValue" : 0,
// "MinValue" : 0,
// "StartValue" : "",
// "ListItems" : ["A", "B", "C"] ,
// "DefaultValue" : "",
// "Value" : "A"
// })
//
// self.params["Parameters"].append({
// "Name": "Para4",
// "DataType" : "Range",
// "NoOfdecimals" : 0,
// "MaxValue" : 100,
// "MinValue" : 50,
// "StartValue" : ">,60",
// "ListItems" : [">", "<"] ,
// "DefaultValue" : "",
// "Value" : ">,52.000000"
// })
// return json.dumps(self.params)
//
// def setParameters(self, json_str: str) -> bool:
//try :
// self.params = json.loads(json_str)
// return True
// except Exception as e :
//print(f"[setParameters] Failed to parse parameters: {e}")
//return False
+255
View File
@@ -0,0 +1,255 @@
#ifndef ANSCUSTOMCODE_H
#define ANSCUSTOMCODE_H
#pragma once
#include <string>
#include <vector>
#include <opencv2/opencv.hpp>
#define CUSTOM_API __declspec(dllexport)
struct CustomObject
{
int classId{ 0 };
int trackId{ 0 };
std::string className{};
float confidence{ 0.0 };
cv::Rect box{};
std::vector<cv::Point2f> polygon; // Polygon that contain x1,y1,x2,y2,x3,y3,x4,y4
cv::Mat mask{}; // Face image in box (cropped) or mask image for segmentation
cv::cuda::GpuMat gpuMask{}; // GPU-resident face crop (set by NV12 affine warp, avoids re-upload)
std::vector<float> kps{}; // Pose exsimate 17 keypoints or oriented bouding box (xcenter, ycenter, width, height, angle)
std::string extraInfo; // More information such as facial recognition
std::string cameraId; // Use to check if this object belongs to any camera
//std::string attributes; // Attributes such as keypoint string
};
struct CustomPoint {
int x, y;
};
struct CustomROIConfig {
bool Rectangle;
bool Polygon;
bool Line;
int MinItems;
int MaxItems;
std::string Name;
std::string ROIMatch;
};
// Data type for custom parameters
// DataType can be Boolean, Integer,Float,List-Single, List-Multiple, Range, String, DaySelections
struct CustomParameter {
std::string Name;
std::string DataType;
int NoOfDecimals;
int MaxValue;
int MinValue;
std::string StartValue;
std::vector<std::string> ListItems;
std::string DefaultValue;
std::string Value;
};
struct CustomROIValue {
std::string ROIMatch;
std::vector<CustomPoint> ROIPoints;
std::string Option;
std::string Name;
int OriginalImageSize;
};
struct CustomParams {
std::vector<CustomROIConfig> ROI_Config;
std::vector<std::string> ROI_Options;
std::vector<CustomParameter> Parameters;
std::vector<CustomROIValue> ROI_Values;
};
/* Example
{
"ROI_Config":[
{
"Rectangle":true,
"Polygon":true,
"Line":false,
"MinItems":0,
"MaxItems":3,
"Name":"Traffic Light",
"ROI-Match":"All Corners"
},
{
"Rectangle":true,
"Polygon":false,
"Line":false,
"MinItems":1,
"MaxItems":1,
"Name":"Car Zone",
"ROI-Match":"All Corners"
},
{
"Rectangle":false,
"Polygon":false,
"Line":true,
"MinItems":1,
"MaxItems":2,
"Name":"Cross Line",
"ROI-Match":"All Corners"
}
],
"ROI_Options":[
"Inside ROI",
"Inside ROI",
"Both Directions"
],
"Parameters":[
{
"Name":"Para1",
"DataType":"Boolean",
"NoOfdecimals":0,
"MaxValue":0,
"MinValue":0,
"StartValue":"",
"ListItems":[],
"DefaultValue":"",
"Value":"true"
},
{
"Name":"Para2",
"DataType":"Integer",
"NoOfdecimals":0,
"MaxValue":5,
"MinValue":1,
"StartValue":"2",
"ListItems":[],
"DefaultValue":"",
"Value":"3"
},
{
"Name":"Para3",
"DataType":"List-Single",
"NoOfdecimals":0,
"MaxValue":0,
"MinValue":0,
"StartValue":"",
"ListItems":["A","B","C"],
"DefaultValue":"",
"Value":"A"
},
{
"Name":"Para4",
"DataType":"Range",
"NoOfdecimals":0,
"MaxValue":100,
"MinValue":50,
"StartValue":">,60",
"ListItems":[">","<"],
"DefaultValue":"",
"Value":">,52.000000"
}
],
"ROI_Values":[
{
"ROI-Match":"Centre Point",
"ROIPoints":[
{"x":269,"y":134},
{"x":777,"y":134},
{"x":777,"y":457},
{"x":269,"y":457}
],
"Option":"Inside ROI",
"Name":"Car Zone 1",
"OriginalImageSize":1920
},
{
"ROI-Match":"Centre Point",
"ROIPoints":[{"x":280,"y":613},{"x":1108,"y":280}],
"Option":"Above",
"Name":"Cross Line 1",
"OriginalImageSize":1920
},
{
"ROI-Match":"Centre Point",
"ROIPoints":[{"x":1511,"y":383},{"x":1283,"y":754}],
"Option":"Left side",
"Name":"Cross Line 2",
"OriginalImageSize":1920
},
{
"ROI-Match":"Centre Point",
"ROIPoints":[
{"x":229,"y":161},
{"x":964,"y":161},
{"x":964,"y":628},
{"x":229,"y":628}
],
"Option":"Left side",
"Name":"Traffic Light 1",
"OriginalImageSize":1920
},
{
"ROI-Match":"Centre Point",
"ROIPoints":[
{"x":1115,"y":304},
{"x":1730,"y":304},
{"x":1730,"y":695},
{"x":1115,"y":695}
],
"Option":"Left side",
"Name":"Traffic Light 2",
"OriginalImageSize":1920
},
{
"ROI-Match":"Centre Point",
"ROIPoints":[
{"x":678,"y":683},
{"x":1217,"y":683},
{"x":1217,"y":1026},
{"x":678,"y":1026}
],
"Option":"Left side",
"Name":"Traffic Light 3",
"OriginalImageSize":1920
}
]
}
*/
class CUSTOM_API IANSCustomClass
{
protected:
std::string _modelDirectory; // The directory where the model is located
float _detectionScoreThreshold{ 0.5 };
CustomParams _params; // Parameters for the model
bool _loadEngineOnCreate{ false }; // Load engine on create
public:
virtual bool Initialize(const std::string& modelDirectory, float detectionScoreThreshold, std::string& labelMap) = 0;
virtual bool OptimizeModel(bool fp16) = 0;
virtual std::vector<CustomObject> RunInference(const cv::Mat& input) = 0;
virtual std::vector<CustomObject> RunInference(const cv::Mat& input, const std::string& camera_id) = 0;
virtual bool ConfigureParameters(CustomParams& param) = 0;
bool SetParameters(const CustomParams& param) {
try {
this->_params.ROI_Config.clear();
for (auto& cf : param.ROI_Config) {
this->_params.ROI_Config.push_back(cf);
}
this->_params.ROI_Options.clear();
for (auto& op : param.ROI_Options) {
this->_params.ROI_Options.push_back(op);
}
this->_params.Parameters.clear();
for (auto& par : param.Parameters) {
this->_params.Parameters.push_back(par);
}
this->_params.ROI_Values.clear();
for (auto& roi : param.ROI_Values) {
this->_params.ROI_Values.push_back(roi);
}
return true;
}
catch (...) {
return false;
}
};
void SetLoadEngineOnCreate(bool loadEngineOnCreate) {
this->_loadEngineOnCreate = loadEngineOnCreate;
}
virtual bool Destroy() = 0;
};
#endif
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+147
View File
@@ -0,0 +1,147 @@
#include "ANSFD.h"
namespace ANSCENTER {
bool ANSFD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
return _gpufaceDetector.OptimizeModel(fp16, optimizedModelFolder);
}
else {
return _cpufaceDetector.OptimizeModel(fp16, optimizedModelFolder);
}
}
bool ANSFD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
return _gpufaceDetector.LoadModel(modelZipFilePath, modelZipPassword);
}
else {
return _cpufaceDetector.LoadModel(modelZipFilePath, modelZipPassword);
}
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFD::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
try {
if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
return _gpufaceDetector.LoadModelFromFolder(licenseKey, modelConfig,modelName, className,modelFolder,labelMap);
}
else {
return _cpufaceDetector.LoadModelFromFolder(licenseKey, modelConfig,modelName, className,modelFolder,labelMap);
}
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
try {
ANSODBase::CheckLicense();
if (!_licenseValid) return false;
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::FACEDETECT;
_modelConfig.detectionType = DetectionType::FACEDETECTOR;
_engineType = ANSLicenseHelper::CheckHardwareInformation();
if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
_isInitialized = _gpufaceDetector.Initialize(licenseKey, _modelConfig, modelZipFilePath, modelZipPassword, labelMap);
return _isInitialized;
}
else {
_isInitialized = _cpufaceDetector.Initialize(licenseKey, _modelConfig, modelZipFilePath, modelZipPassword, labelMap);
return _isInitialized;
}
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSFD::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> ANSFD::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::vector<Object> output;
try {
// 1. Basic system checks
if (!_licenseValid) {
this->_logger.LogError("ANSFD::RunInference", "Invalid License", __FILE__, __LINE__);
return output;
}
if (!_isInitialized) {
this->_logger.LogError("ANSFD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return output;
}
// 2. Input validation
if (input.empty() || input.cols < 10 || input.rows < 10) {
this->_logger.LogError("ANSFD::RunInference", "Invalid input image", __FILE__, __LINE__);
return output;
}
// 3. Run GPU or CPU inference safely
if (_engineType == EngineType::NVIDIA_GPU) {
try {
output = _gpufaceDetector.RunInference(input, camera_id);
}
catch (const std::exception& e) {
this->_logger.LogError("ANSFD::RunInference", std::string("GPU inference failed: ") + e.what(), __FILE__, __LINE__);
}
}
else {
try {
output = _cpufaceDetector.RunInference(input, camera_id);
}
catch (const std::exception& e) {
this->_logger.LogError("ANSFD::RunInference", std::string("CPU inference failed: ") + e.what(), __FILE__, __LINE__);
}
}
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSFD::RunInference", std::string("Unhandled exception: ") + e.what(), __FILE__, __LINE__);
}
catch (...) {
this->_logger.LogFatal("ANSFD::RunInference", "Unknown fatal exception occurred", __FILE__, __LINE__);
}
if (_trackerEnabled) {
output = ApplyTracking(output, camera_id);
if (_stabilizationEnabled) output = StabilizeDetections(output, camera_id);
}
return output;
}
ANSFD::~ANSFD() {
try {
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFaceDetection::~ANSFaceDetection()", e.what(), __FILE__, __LINE__);
}
}
bool ANSFD::Destroy() {
try {
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFaceDetection::Destroy()", e.what(), __FILE__, __LINE__);
return false;
}
}
}
+31
View File
@@ -0,0 +1,31 @@
#ifndef ANSFD_H
#define ANSFD_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
#include "SCRFDFaceDetector.h"
#include "ANSOVFaceDetector.h"
namespace ANSCENTER {
class ANSENGINE_API ANSFD :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSFD();
private:
std::string _modelFilePath;
ANSSCRFDFD _gpufaceDetector;
ANSOVFD _cpufaceDetector;
EngineType _engineType;
};
}
#endif
+1
View File
@@ -0,0 +1 @@
#pragma once
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,220 @@
#pragma once
#include "ANSODEngine.h"
#include "ANSYOLOOD.h"
#include "ANSTENSORRTOD.h"
#include "ANSTENSORRTCL.h"
#include "ANSOPENVINOCL.h"
#include "ANSOPENVINOOD.h"
#include "ANSYOLOV10RTOD.h"
#include "ANSYOLOV10OVOD.h"
#include "ANSCUSTOMDetector.h"
#include "ANSFD.h"
#include "ANSANOMALIB.h"
#include "ANSPOSE.h"
#include "ANSSAM.h"
#include <pipelines/metadata.h>
#include <models/input_data.h>
#include "utils/visualizer.hpp"
#include <turbojpeg.h>
#include <vector>
#include <map>
#include <string>
#include "ANSLibsLoader.h"
namespace ANSCENTER
{
//------------------------------------------------------------------------------------------------------------------------//
// Facial Recognition base class
void ANSFRBase::CheckLicense() {
try {
_licenseValid = ANSCENTER::ANSLicenseHelper::LicenseVerification(_licenseKey, 1004, "ANSFR");//Default productId=1005
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::CheckLicense()", e.what(), __FILE__, __LINE__);
}
}
bool ANSFRBase::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)
{
try {
ANSCENTER::ANSLibsLoader::Initialize();
_modelFolder = "";
_modelConfigFile = "";
_modelFolder.clear();
_modelConfigFile.clear();
// 0. Check if the modelZipFilePath exist?
if (!FileExist(modelZipFilePath)) {
this->_logger.LogFatal("ANSFRBase::LoadModel", "Model zip file is not exist", __FILE__, __LINE__);
}
// 1. Unzip model zip file to a special location with folder name as model file (and version)
std::string outputFolder;
std::vector<std::string> passwordArray;
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
passwordArray.push_back("AnsCustomModels20@$");
passwordArray.push_back("AnsDemoModels20@!");
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
//this->_logger.LogDebug("ANSFRBase::LoadModel. Model name", modelName, __FILE__, __LINE__);
size_t vectorSize = passwordArray.size();
for (size_t i = 0; i < vectorSize; i++) {
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
break; // Break the loop when the condition is met.
}
// 2. Check if the outputFolder exist
if (!std::filesystem::exists(_modelFolder)) {
this->_logger.LogError("ANSFRBase::LoadModel. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
return false; // That means the model file is not exist or the password is not correct
}
// 3. Check if the model has the configuration file
std::string modelConfigName = "model_config.json";
_modelConfigFile = CreateFilePath(_modelFolder, modelConfigName);
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFRBase::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
try {
ANSCENTER::ANSLibsLoader::Initialize();
_licenseKey = licenseKey;
_licenseValid = false;
_modelFolder = "";
_modelConfigFile = "";
_modelFolder.clear();
_modelConfigFile.clear();
CheckLicense();
if (!_licenseValid) {
this->_logger.LogError("ANSFRBase::Initialize", "Invalid License", __FILE__, __LINE__);
return false;
}
// 0. Check if the modelZipFilePath exist?
if (!FileExist(modelZipFilePath)) {
this->_logger.LogFatal("ANSFRBase::Initialize", "Model zip file is not exist", __FILE__, __LINE__);
}
// 1. Unzip model zip file to a special location with folder name as model file (and version)
std::string outputFolder;
std::vector<std::string> passwordArray;
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
passwordArray.push_back("AnsCustomModels20@$");
passwordArray.push_back("AnsDemoModels20@!");
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
//this->_logger.LogDebug("ANSFRBase::Initialize. Model name", modelName, __FILE__, __LINE__);
size_t vectorSize = passwordArray.size();
for (size_t i = 0; i < vectorSize; i++) {
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
break; // Break the loop when the condition is met.
}
// 2. Check if the outputFolder exist
if (!std::filesystem::exists(_modelFolder)) {
this->_logger.LogError("ANSFRBase::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
return false; // That means the model file is not exist or the password is not correct
}
// 3. Check if the model has the configuration file
std::string modelConfigName = "model_config.json";
_modelConfigFile = CreateFilePath(_modelFolder, modelConfigName);
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSFRBase::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
try {
ANSCENTER::ANSLibsLoader::Initialize();
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::OptimizeModel", e.what(), __FILE__, __LINE__);
return false;
}
}
ANSFRBase::~ANSFRBase() {
try {
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::~ANSFRBase", e.what(), __FILE__, __LINE__);
}
}
std::vector<float> ANSFRBase::L2Normalize(const std::vector<float>& values) {
try {
size_t num_val = values.size();
if (num_val == 0) {
return {};
}
std::vector<float> norm(num_val);
float l2_sum_val = 0.f;
for (const auto& value : values) {
l2_sum_val += (value * value);
}
float l2_sum_sqrt = std::sqrt(l2_sum_val);
// Check for zero division using machine epsilon
if (std::abs(l2_sum_sqrt) < std::numeric_limits<float>::epsilon())
{
// Return a default value or handle the error as needed
return std::vector<float>(num_val, 0.0f);
}
for (size_t i = 0; i < num_val; ++i) {
norm[i] = values[i] / l2_sum_sqrt;
}
return norm;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::L2Normalize", e.what(), __FILE__, __LINE__);
return {};
}
}
float ANSFRBase::CosineSimilarity(const std::vector<float>& a, const std::vector<float>& b, bool normalized) {
try {
if (a.size() != b.size() || a.empty()) {
// Return a default value or handle the error as needed
return 1.0f;
}
size_t num_val = a.size();
float mul_a = 0.f, mul_b = 0.f, mul_ab = 0.f;
if (normalized) {
for (size_t i = 0; i < num_val; ++i) {
mul_a += (a[i] * a[i]);
mul_b += (b[i] * b[i]);
mul_ab += (a[i] * b[i]);
}
}
else {
auto norm_a = L2Normalize(a);
auto norm_b = L2Normalize(b);
for (size_t i = 0; i < num_val; ++i) {
mul_a += (norm_a[i] * norm_a[i]);
mul_b += (norm_b[i] * norm_b[i]);
mul_ab += (norm_a[i] * norm_b[i]);
}
}
// Check for zero division using machine epsilon
float denominator = std::sqrt(mul_a) * std::sqrt(mul_b);
if (std::abs(denominator) < std::numeric_limits<float>::epsilon()) {
// Return a default value or handle the error as needed
return 1.0f;
}
return (mul_ab / denominator);
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSFRBase::CosineSimilarity", e.what(), __FILE__, __LINE__);
return 1.0f;
}
}
}
File diff suppressed because it is too large Load Diff
+295
View File
@@ -0,0 +1,295 @@
#ifndef ANSFIRENSMOKE_H
#define ANSFIRENSMOKE_H
#pragma once
#include "ANSEngineCommon.h"
#include <opencv2/opencv.hpp>
#include <opencv2/dnn.hpp>
#include <openvino/openvino.hpp>
#include "ANSYOLOV10OVOD.h"
#include "ANSYOLOV10RTOD.h"
namespace ANSCENTER
{
class ImageProcessor {
const int MAX_QUEUE_SIZE = 10;
const int THRESHOLD = 8;
public:
// Add image to the queue and process if queue has 10 images
std::vector<cv::Rect> addImageToQueue(const cv::Mat& image) {
width = image.cols;
height = image.rows;
// Check if queue size is already 10, if so, remove the oldest image
if (imageQueue.size() == MAX_QUEUE_SIZE) {
imageQueue.pop();
}
// Add the new image to the queue
imageQueue.push(image.clone());
// Process images if we have exactly 10 images in the queue
if (imageQueue.size() == MAX_QUEUE_SIZE) {
return processQueueImages();
}
// Return an empty vector if not yet ready to process
return std::vector<cv::Rect>();
}
private:
int width, height;
std::queue<cv::Mat> imageQueue;
// Sum all images in the queue, create a mask, and return array of bounding rectangles
std::vector<cv::Rect> processQueueImages() {
// Initialize the sum image with zeros
cv::Mat sumImage = cv::Mat::zeros(height, width, CV_32FC1);
// Use a temporary queue to preserve the original images
std::queue<cv::Mat> tempQueue = imageQueue;
// Sum up all images in the queue
while (!tempQueue.empty()) {
cv::Mat img = tempQueue.front();
tempQueue.pop();
// Accumulate image pixels in sumImage
sumImage += img;
}
// Threshold the summed image to create a binary mask
cv::Mat mask;
cv::threshold(sumImage, mask, THRESHOLD, 1, cv::THRESH_BINARY);
// Convert mask to 8-bit to find contours
mask.convertTo(mask, CV_8UC1);
// Find contours on the mask
std::vector<std::vector<cv::Point>> contours;
cv::findContours(mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
// Extract bounding rectangles for each contour
std::vector<cv::Rect> boundingRects;
for (const auto& contour : contours) {
cv::Rect boundingRect = cv::boundingRect(contour);
boundingRects.push_back(boundingRect);
}
return boundingRects;
}
};
class GLCM {
public:
explicit GLCM(const std::string& channel) : channel(channel) {}
std::vector<cv::Mat> getGLCM(const cv::Mat& imageChannel,
const std::vector<int>& distances = { 5 },
const std::vector<float>& angles = { 0.0f },// { 0.0f, CV_PI / 4.0f, CV_PI / 2.0f, 3.0f * CV_PI / 4.0f },
int levels = 256,
bool symmetric = false,
bool normed = false) {
cv::Mat imageChannel8;
imageChannel.convertTo(imageChannel8, CV_8U);
return graycomatrix(imageChannel8, distances, angles, levels, symmetric, normed);
}
void getStockPropFromGLCM(const std::vector<cv::Mat>& glcmMatrices, const std::string& prop) {
auto featureVector = graycoprops(glcmMatrices, prop);
for (size_t i = 0; i < featureVector.size(); ++i) {
props[prop + std::to_string(i + 1) + "_channel" + channel] = featureVector[i];
}
}
const std::map<std::string, double>& getProps() const {
return props;
}
private:
std::string channel;
std::map<std::string, double> props;
void normalizeGLCM(cv::Mat& glcm);
std::vector<cv::Mat> graycomatrix(const cv::Mat& image,
const std::vector<int>& distances,
const std::vector<float>& angles,
int levels = 256,
bool symmetric = false,
bool normed = false);
std::vector<double> graycoprops(const std::vector<cv::Mat>& glcmMatrices, const std::string& property);
};
class HaralickFeatureExtractor {
public:
HaralickFeatureExtractor() {
}
~HaralickFeatureExtractor()
{
}
void Init(const std::string& colourSpace = "RGB",
const std::vector<std::string>& properties = { "energy","homogeneity","dissimilarity","contrast"})
{
props = properties;
for (char channel : colourSpace) {
channels.push_back(std::string(1, channel));
}
}
cv::Mat resizeWithMinAspectRatio(const cv::Mat& inputImage, int minSize) {
// Get original dimensions
int originalWidth = inputImage.cols;
int originalHeight = inputImage.rows;
if ((originalWidth > minSize) && (originalHeight > minSize))return inputImage;
// Calculate aspect ratio
float aspectRatio = static_cast<float>(originalWidth) / static_cast<float>(originalHeight);
// Calculate new dimensions based on the minimum size
int newWidth, newHeight;
if (originalWidth > originalHeight) {
newHeight = minSize;
newWidth = static_cast<int>(minSize * aspectRatio);
}
else {
newWidth = minSize;
newHeight = static_cast<int>(minSize / aspectRatio);
}
// Resize the image
cv::Mat resizedImage;
cv::resize(inputImage, resizedImage, cv::Size(newWidth, newHeight));
return resizedImage;
}
std::vector<double> createFeatureVector(const cv::Mat& inputImage) {
//0. Resize the image to a minimum size
cv::Mat frame = resizeWithMinAspectRatio(inputImage, 200);// min is 20
//1. Convert to RGB format if necessary
cv::Mat imageArray;
if (inputImage.channels() == 3) {
cv::cvtColor(frame, imageArray, cv::COLOR_BGR2RGB); // Convert BGR to RGB
}
else {
return std::vector<double>();
}
// Split the image into individual channels
std::vector<cv::Mat> imageChannels(3);
cv::split(imageArray, imageChannels);
// Initialize GLCM objects for each channel
GLCM glcmChannel0("R"); // Red channel
GLCM glcmChannel1("G"); // Green channel
GLCM glcmChannel2("B"); // Blue channel
// Calculate GLCM matrices for each channel
auto glcmChannel0Matx = glcmChannel0.getGLCM(imageChannels[0]);
auto glcmChannel1Matx = glcmChannel1.getGLCM(imageChannels[1]);
auto glcmChannel2Matx = glcmChannel2.getGLCM(imageChannels[2]);
// Compute properties for each channel
for (const auto& prop : props) {
glcmChannel0.getStockPropFromGLCM(glcmChannel0Matx, prop);
glcmChannel1.getStockPropFromGLCM(glcmChannel1Matx, prop);
glcmChannel2.getStockPropFromGLCM(glcmChannel2Matx, prop);
}
// Combine all properties into a single feature vector in the desired order
std::vector<double> featureVector;
// Iterate over channels first (R, G, B)
for (size_t channelIdx = 0; channelIdx < 3; ++channelIdx) {
const GLCM* glcmChannel = nullptr;
std::string channelName;
// Select the appropriate channel
if (channelIdx == 0) {
glcmChannel = &glcmChannel0;
channelName = "R";
}
else if (channelIdx == 1) {
glcmChannel = &glcmChannel1;
channelName = "G";
}
else if (channelIdx == 2) {
glcmChannel = &glcmChannel2;
channelName = "B";
}
// Retrieve properties for the selected channel
const auto& propsMap = glcmChannel->getProps();
// Iterate over properties for the current channel
for (const auto& prop : props) {
std::string key = prop + "1_channel" + channelName;
if (propsMap.find(key) != propsMap.end()) {
featureVector.push_back(propsMap.at(key));
}
}
}
frame.release();
return featureVector;
}
private:
std::vector<std::string> props;
std::vector<std::string> channels;
};
class ANSENGINE_API ANSFIRENSMOKE :public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSFIRENSMOKE();
private:
double _hsvThreshold;
std::vector<Range> _smoke_colour;
std::vector<Range> _fire_colour;
std::string _classifierModelPath;
ANSOYOLOV10OVOD _cpuObjectDetector;
ANSYOLOV10RTOD _gpuObjectDetector;
EngineType _engineType;
ImageProcessor _smokeImageProcessor;
ImageProcessor _fireImageProcessor;
ANNHUBClassifier _annhub;
const int SMOKE_THRESHOLD_SIZE = 4;
const int HISTORY_SIZE = 30;
const int MIN_MATCHES = 9;
const float IOU_THRESHOLD = 0.4;
const int MINIMUM_STABLE_DURATION = 30;
const float MAX_MOVEMENT = 40.0;
const float ALPHA = 0.7;
cv::Rect _detectedArea;// Area where fire and smoke are detected
cv::Rect _previousDetectedArea;// Area where fire and smoke are detected
int _retainDetectedArea{ 0 };
bool _isFireNSmokeDetected{ false };
bool _classifierInitialized{ false };
HaralickFeatureExtractor _extractor;
std::vector<std::pair<Object, int>> _persistent_detections;
std::deque<std::vector<Object>> _stablization_queue;
private:
bool MajorityColourInFrame(const cv::Mat frame, Range range, float area_threshold = 0.8);
bool MajorityColourInFrame(const cv::Mat frame, std::vector<Range> ranges, float area_threshold = 0.8);
bool DetectFireNSmokeColourInFrame(const cv::Mat& frame, const cv::Rect bBox, const std::vector<Range>& ranges, float area_threshold);
cv::Mat CreateBinaryImageWithRects(int width, int height, const std::vector<cv::Rect>& rects);
cv::Point2f CalculateCentroid(const std::vector<cv::Rect>& group);
cv::Point2f CalculateGroupCenter(const std::vector<cv::Rect>& group);
std::vector<cv::Rect> CreateCoveringMaskRects(const cv::Mat& image, const std::vector<cv::Rect>& maskRects, float maxDistance);
std::vector<Object> ProcecssDetection(const std::vector<Object>& detectedObjects, const std::string& camera_id, int threshold = 8);
cv::Rect CreateMinimumSquareBoundingBox(const std::vector<Object>& detectedObjects, int minSize = 640);
bool IsOverlap(const cv::Rect& target, const std::vector<Object>& rectArray);
bool IsFireDetected(const cv::Mat image, const cv::Rect bBox);
bool IsSmokeDetected(const cv::Mat image, const cv::Rect bBox);
float calculateIOU(const cv::Rect& box_a, const cv::Rect& box_b);
std::vector<Object> StablizeDetection(const std::vector<Object>& detectedObjects, const std::string& camera_id);
};
}
#endif
@@ -0,0 +1,23 @@
// ANSGpuFrameRegistry.cpp — Cross-DLL singleton resolver for ANSODEngine.dll.
//
// Finds the canonical ANSGpuFrameRegistry instance exported by ANSCV.dll
// via GetProcAddress. No link dependency on ANSCV.lib needed.
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include "ANSGpuFrameRegistry.h"
ANSGpuFrameRegistry* ANSGpuFrameRegistry::resolveProcessWide() {
// ANSCV.dll is always loaded before inference starts (it provides RTSP).
HMODULE hMod = GetModuleHandleA("ANSCV.dll");
if (hMod) {
typedef ANSGpuFrameRegistry* (*GetInstanceFn)();
auto fn = reinterpret_cast<GetInstanceFn>(
GetProcAddress(hMod, "ANSGpuFrameRegistry_GetInstance"));
if (fn) return fn();
}
// Fallback: local instance (unit tests without ANSCV.dll).
static ANSGpuFrameRegistry local;
return &local;
}
File diff suppressed because it is too large Load Diff
+247
View File
@@ -0,0 +1,247 @@
#ifndef ANSMOTIONDETECTOR_H
#define ANSMOTIONDETECTOR_H
#pragma once
#include "ANSEngineCommon.h"
#include <string>
#include <opencv2/opencv.hpp>
#include <openvino/openvino.hpp>
#include <array>
namespace ANSCENTER
{
class ANSENGINE_API ANSMOTIONDETECTOR : public ANSODBase
{
protected:
struct CameraStats {
size_t total_frames_processed = 0;
size_t frames_with_movement = 0;
size_t frames_rejected_by_temporal_check = 0;
size_t control_frames_count = 0;
double average_psnr = 0.0;
double min_psnr = std::numeric_limits<double>::max();
double max_psnr = 0.0;
double average_temporal_consistency = 0.0;
std::chrono::milliseconds total_processing_time{ 0 };
std::chrono::high_resolution_clock::time_point last_movement_time;
// Reset stats
void reset()
{
total_frames_processed = 0;
frames_with_movement = 0;
frames_rejected_by_temporal_check = 0;
control_frames_count = 0;
average_psnr = 0.0;
min_psnr = std::numeric_limits<double>::max();
max_psnr = 0.0;
average_temporal_consistency = 0.0;
total_processing_time = std::chrono::milliseconds{ 0 };
}
};
struct CameraData
{
// Detection state
bool movement_detected = false;
bool transition_detected = false;
size_t next_frame_index = 0;
size_t next_key_frame = 0;
double most_recent_psnr_score = 0.0;
size_t frame_index_with_movement = 0;
double max_change_percentage = 15.0;
double min_change_percentage = 0.5;
std::chrono::high_resolution_clock::time_point movement_last_detected;
// Control frames storage
std::map<size_t, cv::Mat> control_frames;
// Output data
cv::Mat output;
cv::Mat mask;
std::vector<std::vector<cv::Point>> contours;
// Configuration parameters
size_t key_frame_frequency = 20;
size_t number_of_control_frames = 5; // CHANGED: Reduce from 10 to 5
double psnr_threshold = 35;
double thumbnail_ratio = 0.1;
cv::Size thumbnail_size = cv::Size(0, 0);
// NEW: Maximum age of control frames to compare (in frames)
size_t max_control_frame_age = 20; // Only compare frames within last 100 frames (~5 seconds at 20fps)
// NEW: Sensitivity multiplier (1.0 = normal, 1.2 = more sensitive, 0.8 = less sensitive)
double sensitivity_multiplier = 1.1;
// Mask computation parameters
int mask_diff_threshold = 10; // Fixed threshold for difference detection
double mask_min_percentage = 0.01; // Min % of frame to be valid movement
double mask_max_percentage = 40.0; // Max % of frame (reject as scene change)
double min_mask_pixels_percentage = 0.05; // 0.02% of working image
// Filtering parameters
double min_object_area = 1000;
int min_object_dimension = 5;
int min_object_total_size = 25;
// Morphology parameters
int morphology_iterations = 2;
// Temporal consistency parameters
bool temporal_consistency_enabled = true;
double mask_overlap_threshold = 0.1;
size_t temporal_history_size = 5;
size_t min_consistent_frames = 3;
bool location_stability_enabled = true;
double max_location_jitter = 30.0;
// Temporal consistency state
std::deque<cv::Mat> mask_history;
std::deque<cv::Point> centroid_history;
size_t consistent_frame_count = 0;
double last_temporal_consistency_score = 0.0;
// Statistics
CameraStats stats;
// Clear function to release memory
void clear()
{
for (auto& [index, frame] : control_frames)
{
frame.release();
}
control_frames.clear();
output.release();
mask.release();
contours.clear();
// Clear temporal history
for (auto& m : mask_history)
{
m.release();
}
mask_history.clear();
centroid_history.clear();
// Reset state
movement_detected = false;
transition_detected = false;
most_recent_psnr_score = 0.0;
frame_index_with_movement = 0;
thumbnail_size = cv::Size(0, 0);
consistent_frame_count = 0;
last_temporal_consistency_score = 0.0;
}
};
// Private member functions
double psnr(const cv::Mat& src, const cv::Mat& dst);
cv::Mat simple_colour_balance(const cv::Mat& src);
cv::Rect BoundingBoxFromContour(const std::vector<cv::Point>& contour);
void setDynamicPSNRThreshold(CameraData& camera, const cv::Mat& working_image);
// Multi-camera data storage
std::unordered_map<std::string, CameraData> cameras;
mutable std::recursive_mutex cameras_mutex;
// Helper functions
CameraData& getCameraData(const std::string& camera_id);
const CameraData* getCameraDataConst(const std::string& camera_id) const;
bool cameraExists(const std::string& camera_id) const;
// Processing helpers
cv::Mat computeMovementMask(const cv::Mat& control_frame, const cv::Mat& current_frame,
const cv::Size& output_size, int morphology_iterations, const CameraData& camera);
std::vector<Object> extractObjectsFromMask(const cv::Mat& mask, const cv::Mat& image,
CameraData& camera, const std::string& camera_id);
void updateControlFrames(CameraData& camera, size_t frame_index, const cv::Mat& thumbnail);
void updateStatistics(CameraData& camera, double psnr, bool movement_detected,
std::chrono::milliseconds processing_time);
// Temporal consistency helpers
bool checkTemporalConsistency(CameraData& camera, const cv::Mat& current_mask);
double calculateMaskOverlap(const cv::Mat& mask1, const cv::Mat& mask2);
cv::Point calculateMaskCentroid(const cv::Mat& mask);
double calculateLocationStability(const std::deque<cv::Point>& centroids);
void updateTemporalHistory(CameraData& camera, const cv::Mat& mask);
std::vector<Object> MovementDetectInternal(const std::string& camera_id,const size_t frame_index,const cv::Mat& image,CameraData& camera);
// Main detection methods
std::vector<Object> MovementDetect(const std::string& camera_id, const cv::Mat& next_image);
std::vector<Object> MovementDetect(const std::string& camera_id, const size_t frame_index, const cv::Mat& image);
// Camera management
bool hasCameraData(const std::string& camera_id) const;
void removeCamera(const std::string& camera_id);
std::vector<std::string> getCameraIds() const;
// Configuration methods
void setThreshold(const std::string& camera_id, double threshold);
void setKeyFrameFrequency(const std::string& camera_id, size_t frequency);
void setNumberOfControlFrames(const std::string& camera_id, size_t count);
void setThumbnailRatio(const std::string& camera_id, double ratio);
//void setMaskEnabled(const std::string& camera_id, bool enabled);
//void setContoursEnabled(const std::string& camera_id, bool enabled);
//void setBboxEnabled(const std::string& camera_id, bool enabled);
//void setContourThickness(const std::string& camera_id, int thickness);
//void setBboxThickness(const std::string& camera_id, int thickness);
void setMinObjectArea(const std::string& camera_id, double area);
void setMinObjectSize(const std::string& camera_id, int size);
void setMorphologyIterations(const std::string& camera_id, int iterations);
// Temporal consistency settings
void setTemporalConsistency(const std::string& camera_id, bool enabled);
void setMaskOverlapThreshold(const std::string& camera_id, double threshold);
void setTemporalHistorySize(const std::string& camera_id, size_t size);
void setMinConsistentFrames(const std::string& camera_id, size_t frames);
void setLocationStabilityEnabled(const std::string& camera_id, bool enabled);
void setMaxLocationJitter(const std::string& camera_id, double pixels);
// Getters for configuration
double getThreshold(const std::string& camera_id) const;
size_t getKeyFrameFrequency(const std::string& camera_id) const;
size_t getNumberOfControlFrames(const std::string& camera_id) const;
double getThumbnailRatio(const std::string& camera_id) const;
bool isMaskEnabled(const std::string& camera_id) const;
bool isContoursEnabled(const std::string& camera_id) const;
bool isBboxEnabled(const std::string& camera_id) const;
bool isTemporalConsistencyEnabled(const std::string& camera_id) const;
// State query methods
bool isMovementDetected(const std::string& camera_id) const;
bool wasTransitionDetected(const std::string& camera_id) const;
double getPSNRScore(const std::string& camera_id) const;
size_t getFrameIndexWithMovement(const std::string& camera_id) const;
std::chrono::milliseconds getTimeSinceLastMovement(const std::string& camera_id) const;
size_t getControlFrameCount(const std::string& camera_id) const;
size_t getNextFrameIndex(const std::string& camera_id) const;
double getTemporalConsistencyScore(const std::string& camera_id) const;
// Public utility methods
bool empty(const std::string& camera_id) const;
void clear(const std::string& camera_id);
void clearAll();
/*cv::Mat getOutput(const std::string& camera_id) const;*/
const cv::Mat& getOutput(const std::string& camera_id) const;
const cv::Mat getMask(const std::string& camera_id) const;
std::vector<std::vector<cv::Point>> getContours(const std::string& camera_id) const;
CameraStats getStats(const std::string& camera_id) const;
void resetStats(const std::string& camera_id);
protected:
std::string _modelFilePath;
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSMOTIONDETECTOR();
};
}
#endif
File diff suppressed because it is too large Load Diff
+116
View File
@@ -0,0 +1,116 @@
#ifndef ANSODENGINE_H
#define ANSODENGINE_H
#define ANSODENGINE_API __declspec(dllexport)
#pragma once
#include "ANSEngineCommon.h"
#include "LabVIEWHeader/extcode.h"
extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase** Handle,
const char* licenseKey,
const char* modelFilePath,
const char* modelFileZipPassword,
float detectionScoreThreshold,
float modelConfThreshold,
float modelMNSThreshold,
int autoDetectEngine,
int modelType,
int detectionType, int loadEngineOnCreation=1);
extern "C" ANSODENGINE_API std::string RunInference(ANSCENTER::ANSODBase * *Handle, unsigned char* jpeg_string, unsigned int bufferLength);
extern "C" ANSODENGINE_API std::string RunTiledInference(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, int tiledWidth, int titledHeight, double overlap, const char* cameraId);
extern "C" ANSODENGINE_API std::string RunInferenceFromJpegString(ANSCENTER::ANSODBase** Handle, const char* jpeg_string, unsigned long jpeg_size, const char* cameraId);
extern "C" ANSODENGINE_API std::string RunTiledInferenceFromJpegString(ANSCENTER::ANSODBase** Handle, const char* jpeg_string, unsigned long jpeg_size, int tiledWidth, int titledHeight, double overlap, const char* cameraId);
extern "C" ANSODENGINE_API void RunTiledInferenceFromCV(ANSCENTER::ANSODBase** Handle, cv::Mat image, int tiledWidth, int titledHeight, double overlap, std::vector<ANSCENTER::Object>& results, const char* cameraId);
extern "C" ANSODENGINE_API std::string RunInferenceInCroppedBBoxImages(ANSCENTER::ANSODBase * *Handle, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, const char* strBboxes);
extern "C" ANSODENGINE_API std::string RunInferenceInCroppedPolygonImages(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, const char* strPolygon);
extern "C" ANSODENGINE_API std::string RunInferenceBinary(ANSCENTER::ANSODBase * *Handle, unsigned char* jpeg_bytes, unsigned int width, unsigned int height);
extern "C" ANSODENGINE_API std::string RunInferenceImagePath(ANSCENTER::ANSODBase * *Handle,const char* imageFilePath);
extern "C" ANSODENGINE_API std::string OptimizeModelStr(const char* modelFilePath, const char* modelFileZipPassword, int modelType, int modelDetectionType, int fp16);
extern "C" ANSODENGINE_API std::string RunInferenceFromCV(ANSCENTER::ANSODBase** Handle, cv::Mat image);
extern "C" ANSODENGINE_API void RunDetectMovement(ANSCENTER::ANSODBase** Handle, cv::Mat image, const char* cameraId, std::vector<ANSCENTER::Object>& results);
extern "C" ANSODENGINE_API int ReleaseANSODHandle(ANSCENTER::ANSODBase **Handle);
//// For LabVIEW API
extern "C" ANSODENGINE_API int CreateANSODHandle_LV(ANSCENTER::ANSODBase * *Handle, const char* licenseKey, const char* modelFilePath, const char* modelFileZipPassword, float modelThreshold, float modelConfThreshold, float modelNMSThreshold, int autoDetectEngine,int modelType, int detectionType, int loadEngineOnCreation,LStrHandle labelMap);
extern "C" ANSODENGINE_API int RunInference_LV(ANSCENTER::ANSODBase **Handle, unsigned char* jpeg_string, unsigned int bufferLength,LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunTiledInference_LV(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, int tiledWidth, int titledHeight, double overlap, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunTiledInferenceFromJpegString_LV(ANSCENTER::ANSODBase** Handle, const char* jpeg_string, unsigned long jpeg_size, int tiledWidth, int titledHeight, double overlap, const char* cameraId,LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceFromJpegString_LV(ANSCENTER::ANSODBase** Handle, const char* jpeg_string, unsigned long jpeg_size, const char* cameraId,LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceBinary_LV(ANSCENTER::ANSODBase * *Handle, unsigned char* jpeg_bytes, unsigned int width, unsigned int height, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceImagePath_LV(ANSCENTER::ANSODBase * *Handle, const char* imageFilePath, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int OptimizeModel(const char* modelFilePath, const char* modelFileZipPassword,int modelType,int modelDetectionType, int fp16, LStrHandle optimizedModelFolder);
extern "C" ANSODENGINE_API int RunInferenceInCroppedBBoxImages_LV(ANSCENTER::ANSODBase * *Handle, unsigned char* jpeg_string, int32 bufferLength, const char* cameraId,const char* strBboxes, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceInCroppedBBoxPolygonImages_LV(ANSCENTER::ANSODBase * *Handle, unsigned char* jpeg_string, int32 bufferLength, const char* cameraId,const char* strPolygon, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunDetectMovement_LV(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceFromJpegStringWithCameraId_LV(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceComplete_LV(ANSCENTER::ANSODBase** Handle,cv::Mat** cvImage , const char* cameraId, int getJpegString, int jpegImageSize, const char* activeROIMode, LStrHandle detectionResult, LStrHandle imageStr);
extern "C" ANSODENGINE_API int RunInferenceComplete_CPP(ANSCENTER::ANSODBase** Handle, cv::Mat** cvImage, const char* cameraId, const char* activeROIMode,std::vector<ANSCENTER::Object>& detectionResult);
extern "C" ANSODENGINE_API int RunInference_CPP(ANSCENTER::ANSODBase** Handle, cv::Mat** cvImage, const char* cameraId, std::vector<ANSCENTER::Object>& detectionResult);
extern "C" ANSODENGINE_API int ShutdownPythonEngine();
extern "C" ANSODENGINE_API int ShutdownPythonEngine_CPP();
// For C# API
extern "C" __declspec(dllexport) const char* CreateANSODHandle_CS(ANSCENTER::ANSODBase** Handle, const char* licenseKey, const char* modelFilePath,
const char* modelFileZipPassword, float modelThreshold, float modelConfThreshold,
float modelNMSThreshold, int autoDetectEngine, int modelType, int detectionType, int loadEngineOnCreation);
extern "C" __declspec(dllexport) const char* RunInferenceImagePath_CS(ANSCENTER::ANSODBase** Handle, const char* imageFilePath);
extern "C" __declspec(dllexport) const char* RunInference_CS(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength);
extern "C" __declspec(dllexport) const char* RunInferenceInCroppedBBoxImages_CS(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId,const char* strBboxes);
extern "C" __declspec(dllexport) const char* RunInferenceInCroppedPolygonImages_CS(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, const char* strPolygon);
extern "C" __declspec(dllexport) const char* RunInferenceBinary_CS(ANSCENTER::ANSODBase** Handle, unsigned char* jpeg_bytes, unsigned int width, unsigned int height);
extern "C" __declspec(dllexport) const char* OptimizeModelStr_CS(const char* modelFilePath, const char* modelFileZipPassword, int modelType, int modelDetectionType, int fp16);
// Utilities
extern "C" __declspec(dllexport) int GetEngineType();
extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase** Handle, const char* licenseKey,
const char* modelName,
const char* className,
float detectionScoreThreshold,
float modelConfThreshold,
float modelMNSThreshold,
int autoDetectEngine,
int modelType,
int detectionType,
int loadEngineOnCreation,
const char* modelFolder,
std::string&labelMap);
extern "C" __declspec(dllexport) int GetActiveRect(ANSCENTER::ANSODBase** Handle,cv::Mat cvImage, cv::Rect &activeWindow);
extern "C" __declspec(dllexport) int DetectMovement(ANSCENTER::ANSODBase** Handle, cv::Mat image, const char* cameraId, std::vector<ANSCENTER::Object>& results);
extern "C" __declspec(dllexport) int Optimize(ANSCENTER::ANSODBase** Handle,bool fp16);
extern "C" __declspec(dllexport) int SetODParameters(ANSCENTER::ANSODBase** Handle, const char* parameters);
extern "C" __declspec(dllexport) int GetODParameters(ANSCENTER::ANSODBase** Handle, ANSCENTER::Params &param);
extern "C" __declspec(dllexport) int GetConfiguredParameters(ANSCENTER::ANSODBase** Handle, LStrHandle stParam);
extern "C" __declspec(dllexport) int GetConfiguredParameters_CPP(ANSCENTER::ANSODBase** Handle, std::string& stParam);
extern "C" __declspec(dllexport) int UpdateDetectionMinScore(ANSCENTER::ANSODBase** Handle, float detectionScore);
//std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap
extern "C" __declspec(dllexport) int SetPrompt(ANSCENTER::ANSODBase** Handle, const char* textPrompt);
extern "C" __declspec(dllexport) int SetTracker(ANSCENTER::ANSODBase** Handle,int trackerType, int enableTracker);
extern "C" __declspec(dllexport) int SetTrackerParameters(ANSCENTER::ANSODBase** Handle, const char* trackerParams);
extern "C" __declspec(dllexport) int SetStabilizationParameters(ANSCENTER::ANSODBase** Handle, const char* stabParams);
// Version 2
extern "C" ANSODENGINE_API int RunInferenceComplete_LV_V2(
uint64_t handleVal,
cv::Mat** cvImage,
const char* cameraId,
int getJpegString,
int jpegImageSize,
const char* activeROIMode,
LStrHandle detectionResult,
LStrHandle imageStr);
extern "C" ANSODENGINE_API int RunTiledInference_LV_V2(uint64_t handleVal, unsigned char* jpeg_string, unsigned int bufferLength, int tiledWidth, int titledHeight, double overlap, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunTiledInferenceFromJpegString_LV_V2(uint64_t handleVal, const char* jpeg_string, unsigned long jpeg_size, int tiledWidth, int titledHeight, double overlap, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceFromJpegString_LV_V2(uint64_t handleVal, const char* jpeg_string, unsigned long jpeg_size, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceBinary_LV_V2(uint64_t handleVal, unsigned char* jpeg_bytes, unsigned int width, unsigned int height, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceImagePath_LV_V2(uint64_t handleVal, const char* imageFilePath, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceInCroppedBBoxImages_LV_V2(uint64_t handleVal, unsigned char* jpeg_string, int32 bufferLength, const char* cameraId, const char* strBboxes, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceFromJpegStringWithCameraId_LV_V2(uint64_t handleVal, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunDetectMovement_LV_V2(uint64_t handleVal, unsigned char* jpeg_string, unsigned int bufferLength, const char* cameraId, LStrHandle detectionResult);
extern "C" ANSODENGINE_API int RunInferenceInCroppedBBoxPolygonImages_LV_V2(uint64_t handleVal, unsigned char* jpeg_string, int32 bufferLength, const char* cameraId, const char* strPolygon, LStrHandle detectionResult);
#endif
+443
View File
@@ -0,0 +1,443 @@
#include "ANSODHUB.h"
#include "Utility.h"
namespace ANSCENTER
{
// ODHUBAPI
bool ODHUBAPI::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
optimizedModelFolder = GetParentFolder(_modelFilePath);
this->_logger.LogDebug("ODHUBAPI::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
return true;
}
else {
optimizedModelFolder = "";
this->_logger.LogFatal("ODHUBAPI::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
return false;
}
}
bool ODHUBAPI::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// 0. Check weigth file
std::string weightfile = CreateFilePath(_modelFolder, "train_last.weights");
if (FileExist(weightfile)) {
// This is the darknet Yolov4
_modelFilePath = weightfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
_modelCfgPath = CreateFilePath(_modelFolder, "test.cfg");
if (std::filesystem::exists(_modelCfgPath)) {
this->_logger.LogDebug("ODHUBAPI::LoadModel. Loading ODHUB weight", _modelCfgPath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ODHUBAPI::LoadModel. Model config file is not exist", _modelCfgPath, __FILE__, __LINE__);
return false;
}
}
else
{
this->_logger.LogError("ODHUBAPI::LoadModel. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
LoadClassesFromString();
}
else {
LoadClassesFromFile();
}
//1. Load the model
EngineType engineType = ANSLicenseHelper::CheckHardwareInformation();
if (engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
_useGPU = true;
_net = new Detector();
_net->Init(_modelCfgPath.c_str(), _modelFilePath.c_str(), 0, 1);
}
else {
_useGPU = false;
_netCPU = new CPUDetector();
_netCPU->Init(_modelCfgPath.c_str(), _modelFilePath.c_str(), -1, 1);
}
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ODHUBAPI::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ODHUBAPI::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".weights";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.inpHeight = 416;
_modelConfig.inpWidth = 416;
// 0. Check weigth file
std::string weightfile = CreateFilePath(_modelFolder, modelFullName);
if (FileExist(weightfile)) {
// This is the darknet Yolov4
_modelFilePath = weightfile;
_classFilePath = CreateFilePath(_modelFolder, className);
_modelCfgPath = CreateFilePath(_modelFolder, "test.cfg");
if (std::filesystem::exists(_modelCfgPath)) {
this->_logger.LogDebug("ODHUBAPI::LoadModel. Loading ODHUB weight", _modelCfgPath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ODHUBAPI::LoadModel. Model config file is not exist", _modelCfgPath, __FILE__, __LINE__);
return false;
}
}
else
{
this->_logger.LogError("ODHUBAPI::LoadModel. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
LoadClassesFromString();
}
else {
LoadClassesFromFile();
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
//2. Load the model
EngineType engineType = ANSLicenseHelper::CheckHardwareInformation();
if (engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
_useGPU = true;
_net = new Detector();
_net->Init(_modelCfgPath.c_str(), _modelFilePath.c_str(), 0, 1);
}
else {
_useGPU = false;
_netCPU = new CPUDetector();
_netCPU->Init(_modelCfgPath.c_str(), _modelFilePath.c_str(), -1, 1);
}
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ODHUBAPI::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ODHUBAPI::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.inpHeight = 416;
_modelConfig.inpWidth = 416;
// 0. Check weigth file
std::string weightfile = CreateFilePath(_modelFolder, "train_last.weights");
if (FileExist(weightfile)) {
// This is the darknet Yolov4
_modelFilePath = weightfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
_modelCfgPath = CreateFilePath(_modelFolder, "test.cfg");
if (std::filesystem::exists(_modelCfgPath)) {
this->_logger.LogDebug("ODHUBAPI::LoadModel. Loading ODHUB weight", _modelCfgPath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ODHUBAPI::LoadModel. Model config file is not exist", _modelCfgPath, __FILE__, __LINE__);
return false;
}
}
else
{
this->_logger.LogError("ODHUBAPI::LoadModel. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
LoadClassesFromString();
}
else {
LoadClassesFromFile();
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
//2. Load the model
EngineType engineType = ANSLicenseHelper::CheckHardwareInformation();
if (engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
_useGPU = true;
_net = new Detector();
_net->Init(_modelCfgPath.c_str(), _modelFilePath.c_str(), 0, 1);
}
else {
_useGPU = false;
_netCPU = new CPUDetector();
_netCPU->Init(_modelCfgPath.c_str(), _modelFilePath.c_str(), -1, 1);
}
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ODHUBAPI::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ODHUBAPI::RunInference(const cv::Mat& input) {
return RunInference(input, "ODHUBCam");
}
std::vector<Object> ODHUBAPI::RunInference(const cv::Mat& input, const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<Object> result;
try {
// Early validation
if (!_licenseValid) {
_logger.LogError("ODHUBAPI::RunInference", "Invalid License",
__FILE__, __LINE__);
return result;
}
if (!_isInitialized) {
_logger.LogError("ODHUBAPI::RunInference", "Model is not initialized",
__FILE__, __LINE__);
return result;
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
return result;
}
// Prepare image for inference
cv::Mat processedImage;
if (input.channels() == 1) {
cv::cvtColor(input, processedImage, cv::COLOR_GRAY2BGR);
}
else {
// Shallow copy is safe if detect() doesn't modify input
// If it does, use: processedImage = input.clone();
processedImage = input;
}
// Get detections (unified call)
std::vector<bbox_t> boxes = _useGPU
? _net->detect(processedImage, _modelConfig.detectionScoreThreshold, false)
: _netCPU->detect(processedImage, _modelConfig.detectionScoreThreshold, false);
// Pre-allocate result vector
result.reserve(boxes.size());
// Pre-calculate image bounds
const int maxWidth = input.cols;
const int maxHeight = input.rows;
const cv::Rect imageBounds(0, 0, maxWidth, maxHeight);
const int classNameSize = static_cast<int>(_classes.size());
// Convert detections to Objects
for (const auto& box : boxes) {
Object obj;
// Set class info
obj.classId = box.obj_id;
if (!_classes.empty()) {
obj.className = (box.obj_id < classNameSize)
? _classes[box.obj_id]
: _classes[classNameSize - 1];
}
else {
obj.className = "Unknown";
}
// Set detection info
obj.confidence = box.prob;
obj.cameraId = camera_id;
// Clamp bounding box to image bounds
obj.box = cv::Rect(
static_cast<int>(box.x),
static_cast<int>(box.y),
static_cast<int>(box.w),
static_cast<int>(box.h)
) & imageBounds; // Single intersection operation
// Skip invalid boxes (completely outside image)
if (obj.box.area() > 0) {
result.push_back(std::move(obj));
}
}
if (_trackerEnabled) {
result = ApplyTracking(result, camera_id);
if (_stabilizationEnabled) result = StabilizeDetections(result, camera_id);
}
return result;
}
catch (const std::exception& e) {
_logger.LogFatal("ODHUBAPI::RunInference", e.what(), __FILE__, __LINE__);
return {}; // Return empty vector
}
}
ODHUBAPI::~ODHUBAPI() {
try {
this->_logger.LogDebug("ODHUBAPI::~ODHUBAPI()", "Release ODHUBAPI ", __FILE__, __LINE__);
}
catch (std::exception& e) {
this->_logger.LogFatal("ODHUBAPI::~ODHUBAPI()", e.what(), __FILE__, __LINE__);
}
}
bool ODHUBAPI::Destroy() {
try {
this->_logger.LogDebug("ODHUBAPI::Destroy()", "Release ODHUBAPI ", __FILE__, __LINE__);
if (_net != nullptr)delete _net;
if (_netCPU != nullptr)delete _netCPU;
_net = nullptr;
_netCPU = nullptr;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ODHUBAPI::Destroy()", e.what(), __FILE__, __LINE__);
return false;
}
}
}
//std::vector<Object> ODHUBAPI::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> result;
// try {
// if (!_licenseValid) {
// this->_logger.LogError("ODHUBAPI::RunInference", "Invalid License", __FILE__, __LINE__);
// return result;
// }
// if (!_isInitialized) {
// this->_logger.LogError("ODHUBAPI::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return result;
// }
// if (input.empty()) return result;
// if ((input.cols < 10) || (input.rows < 10)) return result;
// int maxWidth = input.cols;
// int maxHeight = input.rows;
// cv::Mat processedImage;
// if (input.channels() == 1) {
// cv::cvtColor(input, processedImage, cv::COLOR_GRAY2BGR);
// }
// else {
// processedImage = input;
// }
// int classNameSize = _classes.size();
// // Run inference
// if (_useGPU) // Use GPU
// {
// std::vector<bbox_t> boxes = _net->detect(processedImage, _modelConfig.detectionScoreThreshold, false);
// for (auto& box : boxes) {
// Object obj;
// // Check if the box is out of the image
// if (box.x < 0) box.x = 0;
// if (box.y < 0) box.y = 0;
// if (box.x + box.w > maxWidth) box.w = maxWidth - box.x - 5;
// if (box.y + box.h > maxHeight) box.h = maxHeight - box.y - 5;
// obj.classId = box.obj_id;
//
// if (!_classes.empty()) {
// if (box.obj_id < classNameSize) {
// obj.className = _classes[box.obj_id];
// }
// else {
// obj.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
// }
// }
// else {
// obj.className = "Unknown"; // Fallback if _classes is empty
// }
// obj.confidence = box.prob;
// obj.box.x = box.x;
// obj.box.y = box.y;
// obj.box.width = box.w;
// obj.box.height = box.h;
// obj.box.x = std::max(0, obj.box.x);
// obj.box.y = std::max(0, obj.box.y);
// obj.box.width = std::min(maxWidth - obj.box.x, obj.box.width);
// obj.box.height = std::min(maxHeight - obj.box.y, obj.box.height);
// obj.cameraId = camera_id;
// result.push_back(obj);
// }
// //EnqueueDetection(result, camera_id);
// return result;
// }
// else // Use CPU
// {
// std::vector<bbox_t> boxes = _netCPU->detect(processedImage, _modelConfig.detectionScoreThreshold, false);
// for (auto& box : boxes) {
// Object obj;
// // Check if the box is out of the image
// if (box.x < 0) box.x = 0;
// if (box.y < 0) box.y = 0;
// if (box.x + box.w > maxWidth) box.w = maxWidth - box.x - 5;
// if (box.y + box.h > maxHeight) box.h = maxHeight - box.y - 5;
// obj.classId = box.obj_id;
// if (!_classes.empty()) {
// if (box.obj_id < classNameSize) {
// obj.className = _classes[box.obj_id];
// }
// else {
// obj.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
// }
// }
// else {
// obj.className = "Unknown"; // Fallback if _classes is empty
// }
// obj.confidence = box.prob;
// obj.box.x = box.x;
// obj.box.y = box.y;
// obj.box.width = box.w;
// obj.box.height = box.h;
// obj.box.x = std::max(0, obj.box.x);
// obj.box.y = std::max(0, obj.box.y);
// obj.box.width = std::min(maxWidth - obj.box.x, obj.box.width);
// obj.box.height = std::min(maxHeight - obj.box.y, obj.box.height);
// obj.cameraId = camera_id;
// result.push_back(obj);
// }
// //EnqueueDetection(result, camera_id);
// return result;
// }
// }
// catch (std::exception& e) {
// result.clear();
// this->_logger.LogFatal("ODHUBAPI::RunInference", e.what(), __FILE__, __LINE__);
// return result;
// }
//}
+39
View File
@@ -0,0 +1,39 @@
#ifndef ANSODHUB_H
#define ANSODHUB_H
#pragma once
#include "ANSEngineCommon.h"
#include <iostream>
#include <iomanip>
#include <string>
#include <vector>
#include <queue>
#include <fstream>
#include <thread>
#include <future>
#include <atomic>
#include <mutex> // std::mutex, std::unique_lock
#include <cmath>
#include "yolo_v2_class.hpp" // imported functions from DLL
#include <yolo_v2cpu_class.hpp>
namespace ANSCENTER {
class ANSENGINE_API ODHUBAPI :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ODHUBAPI();
private:
bool _useGPU;
Detector *_net =nullptr;
CPUDetector *_netCPU=nullptr;
std::string _modelFilePath;
std::string _modelCfgPath;
};
}
#endif
File diff suppressed because it is too large Load Diff
+54
View File
@@ -0,0 +1,54 @@
#ifndef ANSONNXCL_H
#define ANSONNXCL_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
namespace ANSCENTER {
class ANSENGINE_API ANSONNXCL :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSONNXCL();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
size_t vectorProduct(const std::vector<int64_t>& vector);
bool preprocessImageToTensor(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& targetShape,
const cv::Scalar& color = cv::Scalar(0, 0, 0),
bool scaleUp = true,
const std::string& strategy = "resize");
void warmupModel();
bool Init(const std::string& modelPath, const cv::Size& targetInputShape, bool useGPU = true);
bool preprocess(const cv::Mat& image, float*& blob, std::vector<int64_t>& inputTensorShape);
std::vector<Object> postprocess(const std::vector<Ort::Value>& outputTensors, const std::string& camera_id);
std::vector<Object> classify(const cv::Mat& image, const std::string& camera_id);
private:
Ort::Env env_{ nullptr };
Ort::SessionOptions sessionOptions_{ nullptr };
Ort::Session session_{ nullptr };
bool isDynamicInputShape_{};
cv::Size inputImageShape_{};
std::vector<Ort::AllocatedStringPtr> inputNodeNameAllocatedStrings_{};
std::vector<const char*> inputNames_{};
std::vector<Ort::AllocatedStringPtr> outputNodeNameAllocatedStrings_{};
std::vector<const char*> outputNames_{};
size_t numInputNodes_{}, numOutputNodes_{};
int numClasses_{ 0 };
float m_imgWidth = 0;
float m_imgHeight = 0;
};
}
#endif
#pragma once
File diff suppressed because it is too large Load Diff
+108
View File
@@ -0,0 +1,108 @@
#ifndef ANSONNXOBB_H
#define ANSONNXOBB_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
namespace ANSCENTER {
static constexpr float EPS = 1e-7f;
struct OrientedBoundingBox {
float x; // x-coordinate of the center
float y; // y-coordinate of the center
float width; // width of the box
float height; // height of the box
float angle; // rotation angle in radians
OrientedBoundingBox() : x(0), y(0), width(0), height(0), angle(0) {}
OrientedBoundingBox(float x_, float y_, float width_, float height_, float angle_)
: x(x_), y(y_), width(width_), height(height_), angle(angle_) {
}
};
struct Detection {
OrientedBoundingBox box; // Oriented bounding box in xywhr format
float conf{}; // Confidence score
int classId{}; // Class ID
Detection() = default;
Detection(const OrientedBoundingBox& box_, float conf_, int classId_)
: box(box_), conf(conf_), classId(classId_) {
}
};
class ANSENGINE_API ANSONNXOBB :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSONNXOBB();
int getInstanceId() const { return instanceId_; }
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
size_t vectorProduct(const std::vector<int64_t>& vector);
void letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool auto_ = true,
bool scaleFill = false,
bool scaleUp = true,
int stride = 32);
void NMSBoxes(const std::vector<BoundingBox>& boundingBoxes,
const std::vector<float>& scores,
float scoreThreshold,
float nmsThreshold,
std::vector<int>& indices);
void drawBoundingBox(cv::Mat& image, const std::vector<Detection>& detections,
const std::vector<std::string>& classNames, const std::vector<cv::Scalar>& colors);
std::vector<cv::Point2f> OBBToPoints(const OrientedBoundingBox& obb);
std::vector<cv::Scalar> generateColors(const std::vector<std::string>& classNames, int seed = 42);
void getCovarianceComponents(const OrientedBoundingBox& box, float& out1, float& out2, float& out3);
std::vector<std::vector<float>> batchProbiou(const std::vector<OrientedBoundingBox>& obb1, const std::vector<OrientedBoundingBox>& obb2, float eps = EPS);
std::vector<int> nmsRotatedImpl(const std::vector<OrientedBoundingBox>& sorted_boxes, float iou_thres);
std::vector<int> nmsRotated(const std::vector<OrientedBoundingBox>& boxes, const std::vector<float>& scores, float threshold = 0.75f);
std::vector<Object> nonMaxSuppression(
const std::vector<Detection>& input_detections,
const std::string& camera_id,
float conf_thres = 0.25f,
float iou_thres = 0.75f,
int max_det = 1000);
void warmupModel();
bool Init(const std::string& modelPath, bool useGPU=true, int deviceId = 0);
cv::Mat preprocess(const cv::Mat& image, float*& blob, std::vector<int64_t>& inputTensorShape);
std::vector<Object> postprocess(
const cv::Size& originalImageSize,
const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors, int topk,
const std::string& camera_id);
std::vector<Object> detect(const cv::Mat& image, const std::string& camera_id);
private:
static std::atomic<int> instanceCounter_; // Thread-safe counter
int instanceId_;
int deviceId_ = 0;
Ort::Env env{ nullptr }; // ONNX Runtime environment
Ort::SessionOptions sessionOptions{ nullptr }; // Session options for ONNX Runtime
Ort::Session session{ nullptr }; // ONNX Runtime session for running inference
bool isDynamicInputShape{}; // Flag indicating if input shape is dynamic
cv::Size inputImageShape; // Expected input image shape for the model
// Vectors to hold allocated input and output node names
std::vector<Ort::AllocatedStringPtr> inputNodeNameAllocatedStrings;
std::vector<const char*> inputNames;
std::vector<Ort::AllocatedStringPtr> outputNodeNameAllocatedStrings;
std::vector<const char*> outputNames;
size_t numInputNodes, numOutputNodes; // Number of input and output nodes in the model
std::vector<std::string> classNames; // Vector of class names loaded from file
std::vector<cv::Scalar> classColors; // Vector of colors for each class
};
}
#endif
#pragma once
File diff suppressed because it is too large Load Diff
+80
View File
@@ -0,0 +1,80 @@
#ifndef ANSONNXPOSE_H
#define ANSONNXPOSE_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
namespace ANSCENTER {
class ANSENGINE_API ANSONNXPOSE :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSONNXPOSE();
int getInstanceId() const { return instanceId_; }
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
size_t vectorProduct(const std::vector<int64_t>& vector);
void letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool auto_ = true,
bool scaleFill = false,
bool scaleUp = true,
int stride = 32);
void NMSBoxes(const std::vector<BoundingBox>& boundingBoxes,
const std::vector<float>& scores,
float scoreThreshold,
float nmsThreshold,
std::vector<int>& indices);
void drawPoseEstimation(cv::Mat& image,
const std::vector<Object>& detections,
float confidenceThreshold = 0.5,
float kptThreshold = 0.5);
void warmupModel();
bool Init(const std::string& modelPath, bool useGPU=true, int deviceId = 0);
cv::Mat preprocess(const cv::Mat& image, float*& blob, std::vector<int64_t>& inputTensorShape);
std::vector<Object> postprocess(const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors, const std::string& camera_id);
std::vector<Object> detect(const cv::Mat& image, const std::string& camera_id);
private:
static std::atomic<int> instanceCounter_; // Thread-safe counter
int instanceId_;
int deviceId_ = 0;
Ort::Env env{ nullptr }; // ONNX Runtime environment
Ort::SessionOptions sessionOptions{ nullptr }; // Session options for ONNX Runtime
Ort::Session session{ nullptr }; // ONNX Runtime session for running inference
bool isDynamicInputShape{}; // Flag indicating if input shape is dynamic
cv::Size inputImageShape; // Expected input image shape for the model
// Vectors to hold allocated input and output node names
std::vector<Ort::AllocatedStringPtr> inputNodeNameAllocatedStrings;
std::vector<const char*> inputNames;
std::vector<Ort::AllocatedStringPtr> outputNodeNameAllocatedStrings;
std::vector<const char*> outputNames;
size_t numInputNodes, numOutputNodes; // Number of input and output nodes in the model
const std::vector<std::pair<int, int>> POSE_SKELETON = {
// Face connections
{0,1}, {0,2}, {1,3}, {2,4},
// Head-to-shoulder connections
{3,5}, {4,6},
// Arms
{5,7}, {7,9}, {6,8}, {8,10},
// Body
{5,6}, {5,11}, {6,12}, {11,12},
// Legs
{11,13}, {13,15}, {12,14}, {14,16}
};
};
}
#endif
#pragma once
+379
View File
@@ -0,0 +1,379 @@
#include "ANSONNXSAM3.h"
#include "ANSCLIPTokenizer.h"
#include "ONNXSAM3.h"
#include "Utility.h"
namespace ANSCENTER
{
// =========================================================================
// OptimizeModel — ONNX doesn't need separate optimization
// =========================================================================
bool ANSONNXSAM3::OptimizeModel(bool fp16, std::string& optimizedModelFolder)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
// ONNX Runtime handles graph optimization internally.
// Just verify the model folder exists.
std::string imgPath = CreateFilePath(_modelFolder, "sam3_image_encoder.onnx");
if (!FileExist(imgPath)) {
_logger.LogFatal("ANSONNXSAM3::OptimizeModel",
"Model files not found in: " + _modelFolder, __FILE__, __LINE__);
return false;
}
_fp16 = fp16;
optimizedModelFolder = _modelFolder;
return true;
}
// =========================================================================
// Initialize
// =========================================================================
bool ANSONNXSAM3::Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath, const std::string& modelZipPassword,
std::string& labelMap)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::Initialize(licenseKey, modelConfig,
modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
_modelConfig.detectionType = DetectionType::SEGMENTATION;
if (_modelConfig.modelConfThreshold < 0.1f)
_modelConfig.modelConfThreshold = 0.5f;
m_segThreshold = _modelConfig.modelConfThreshold;
// Create ONNX engine from 3-model folder
if (!InitEngine(_modelFolder)) {
_logger.LogError("ANSONNXSAM3::Initialize",
"Failed to init ONNX engine from folder: " + _modelFolder, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
_modelLoadValid = true;
_isInitialized = true;
// Load tokenizer if merges.txt is available
m_tokenizer = std::make_unique<ANSCLIPTokenizer>();
std::string tokenizerPath = CreateFilePath(_modelFolder, "merges.txt");
if (FileExist(tokenizerPath)) {
m_tokenizer->Load(tokenizerPath);
_logger.LogDebug("ANSONNXSAM3::Initialize",
"CLIP tokenizer loaded from: " + tokenizerPath, __FILE__, __LINE__);
}
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXSAM3::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
// =========================================================================
// LoadModel
// =========================================================================
bool ANSONNXSAM3::LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.detectionType = DetectionType::SEGMENTATION;
if (_modelConfig.modelConfThreshold < 0.1f)
_modelConfig.modelConfThreshold = 0.5f;
m_segThreshold = _modelConfig.modelConfThreshold;
// Create ONNX engine from 3-model folder
if (!InitEngine(_modelFolder)) {
_logger.LogError("ANSONNXSAM3::LoadModel",
"Failed to init ONNX engine from folder: " + _modelFolder, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
_modelLoadValid = true;
_isInitialized = true;
// Load tokenizer if merges.txt is available
m_tokenizer = std::make_unique<ANSCLIPTokenizer>();
std::string tokenizerPath = CreateFilePath(_modelFolder, "merges.txt");
if (FileExist(tokenizerPath)) {
m_tokenizer->Load(tokenizerPath);
_logger.LogDebug("ANSONNXSAM3::LoadModel",
"CLIP tokenizer loaded from: " + tokenizerPath, __FILE__, __LINE__);
}
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXSAM3::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
// =========================================================================
// LoadModelFromFolder
// =========================================================================
bool ANSONNXSAM3::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder, std::string& labelMap)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig,
modelName, className, modelFolder, labelMap);
if (!result) return false;
_modelConfig = modelConfig;
_modelConfig.detectionType = DetectionType::SEGMENTATION;
if (_modelConfig.modelConfThreshold < 0.1f)
_modelConfig.modelConfThreshold = 0.5f;
m_segThreshold = _modelConfig.modelConfThreshold;
_modelFilePath = modelFolder;
// Create ONNX engine from 3-model folder
if (!InitEngine(modelFolder)) {
_logger.LogError("ANSONNXSAM3::LoadModelFromFolder",
"Failed to init ONNX engine from folder: " + modelFolder, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
_modelLoadValid = true;
_isInitialized = true;
// Load tokenizer if merges.txt is available
m_tokenizer = std::make_unique<ANSCLIPTokenizer>();
std::string tokenizerPath = CreateFilePath(_modelFolder, "merges.txt");
if (FileExist(tokenizerPath)) {
m_tokenizer->Load(tokenizerPath);
_logger.LogDebug("ANSONNXSAM3::LoadModelFromFolder",
"CLIP tokenizer loaded from: " + tokenizerPath, __FILE__, __LINE__);
}
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXSAM3::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
return false;
}
}
// =========================================================================
// InitEngine — create ONNXSAM3 engine
// =========================================================================
bool ANSONNXSAM3::InitEngine(const std::string& modelFolder)
{
// Verify required model files exist
std::string imgPath = CreateFilePath(modelFolder, "sam3_image_encoder.onnx");
std::string langPath = CreateFilePath(modelFolder, "sam3_language_encoder.onnx");
std::string decPath = CreateFilePath(modelFolder, "sam3_decoder.onnx");
if (!FileExist(imgPath)) {
_logger.LogError("ANSONNXSAM3::InitEngine",
"Image encoder not found: " + imgPath, __FILE__, __LINE__);
return false;
}
if (!FileExist(langPath)) {
_logger.LogError("ANSONNXSAM3::InitEngine",
"Language encoder not found: " + langPath, __FILE__, __LINE__);
return false;
}
if (!FileExist(decPath)) {
_logger.LogError("ANSONNXSAM3::InitEngine",
"Decoder not found: " + decPath, __FILE__, __LINE__);
return false;
}
try {
m_engine = std::make_unique<ONNXSAM3>(modelFolder, EngineType::NVIDIA_GPU);
m_tokenLength = m_engine->getTokenLength();
_logger.LogDebug("ANSONNXSAM3::InitEngine",
"3-session ONNX engine created. inputSize=" + std::to_string(m_engine->getInputSize()) +
" tokenLength=" + std::to_string(m_tokenLength),
__FILE__, __LINE__);
return true;
}
catch (const Ort::Exception& e) {
_logger.LogError("ANSONNXSAM3::InitEngine",
std::string("ORT Exception: ") + e.what(), __FILE__, __LINE__);
return false;
}
catch (const std::exception& e) {
_logger.LogError("ANSONNXSAM3::InitEngine",
std::string("Failed to create ONNX engine: ") + e.what(), __FILE__, __LINE__);
return false;
}
}
// =========================================================================
// SetPrompt (pre-tokenized)
// =========================================================================
bool ANSONNXSAM3::SetPrompt(const std::vector<int64_t>& inputIds,
const std::vector<int64_t>& attentionMask)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (static_cast<int>(inputIds.size()) != m_tokenLength ||
static_cast<int>(attentionMask.size()) != m_tokenLength) {
_logger.LogError("ANSONNXSAM3::SetPrompt",
"Token vectors must have exactly " + std::to_string(m_tokenLength) + " elements",
__FILE__, __LINE__);
return false;
}
if (!m_engine) {
_logger.LogError("ANSONNXSAM3::SetPrompt", "Engine not initialized", __FILE__, __LINE__);
return false;
}
m_engine->setPrompt(inputIds, attentionMask);
m_promptSet = true;
return true;
}
// =========================================================================
// SetPrompt (text string — uses CLIP tokenizer)
// =========================================================================
bool ANSONNXSAM3::SetPrompt(const std::string& text)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!m_tokenizer || !m_tokenizer->IsLoaded()) {
_logger.LogError("ANSONNXSAM3::SetPrompt",
"Tokenizer not loaded. Place merges.txt in model folder, "
"or use SetPrompt(inputIds, attentionMask) directly.",
__FILE__, __LINE__);
return false;
}
auto result = m_tokenizer->Tokenize(text, m_tokenLength);
SetPrompt(result.inputIds, result.attentionMask);
return true;
}
// =========================================================================
// RunInference
// =========================================================================
std::vector<Object> ANSONNXSAM3::RunInference(const cv::Mat& input)
{
return RunInference(input, "");
}
std::vector<Object> ANSONNXSAM3::RunInference(const cv::Mat& input, const std::string& camera_id)
{
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!_modelLoadValid) {
_logger.LogError("ANSONNXSAM3::RunInference", "Model not loaded", __FILE__, __LINE__);
return {};
}
if (!_licenseValid) {
_logger.LogError("ANSONNXSAM3::RunInference", "Invalid license", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("ANSONNXSAM3::RunInference", "Model not initialized", __FILE__, __LINE__);
return {};
}
if (!m_promptSet) {
_logger.LogError("ANSONNXSAM3::RunInference",
"No prompt set. Call SetPrompt() first.", __FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
return {};
}
}
try {
// Run ONNX engine inference
auto sam3Results = m_engine->detect(input, m_segThreshold);
// Convert SAM3Result -> ANSODBase Object
std::vector<Object> results;
results.reserve(sam3Results.size());
const float imgW = static_cast<float>(input.cols);
const float imgH = static_cast<float>(input.rows);
for (auto& sr : sam3Results) {
Object obj;
obj.box = sr.box;
obj.classId = 0;
obj.className = "object";
obj.cameraId = camera_id;
obj.confidence = sr.confidence;
obj.mask = std::move(sr.mask);
// Create normalized polygon from mask (closed, maxPoints-limited)
obj.polygon = ANSUtilityHelper::MaskToNormalizedPolygon(
obj.mask, obj.box, imgW, imgH);
// Fallback: normalized box corners if mask polygon failed
if (obj.polygon.empty()) {
obj.polygon = ANSUtilityHelper::RectToNormalizedPolygon(obj.box, imgW, imgH);
}
results.push_back(std::move(obj));
}
if (_trackerEnabled) {
results = ApplyTracking(results, camera_id);
if (_stabilizationEnabled) results = StabilizeDetections(results, camera_id);
}
return results;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXSAM3::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
// =========================================================================
// Destroy
// =========================================================================
bool ANSONNXSAM3::Destroy()
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
m_engine.reset();
m_tokenizer.reset();
m_promptSet = false;
_modelLoadValid = false;
_isInitialized = false;
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXSAM3::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
ANSONNXSAM3::~ANSONNXSAM3()
{
Destroy();
}
}
+66
View File
@@ -0,0 +1,66 @@
#ifndef ANSONNXSAM3_H
#define ANSONNXSAM3_H
#pragma once
#include "ANSEngineCommon.h"
#include "ANSCLIPTokenizer.h"
#include "ONNXSAM3.h"
namespace ANSCENTER
{
/// ANSONNXSAM3 — text-prompted segmentation using ONNX Runtime.
///
/// Public API (ANSODBase) wrapper around ONNXSAM3 engine.
/// Supports all execution providers via BasicOrtHandler
/// (CUDA, DirectML, OpenVINO, CPU).
class ANSENGINE_API ANSONNXSAM3 : public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath, const std::string& modelZipPassword,
std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder, std::string& labelMap) override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
/// Set the text prompt for segmentation (pre-tokenized).
bool SetPrompt(const std::vector<int64_t>& inputIds,
const std::vector<int64_t>& attentionMask) override;
/// Set the text prompt by tokenizing the given text.
/// Requires merges.txt (CLIP BPE vocabulary) in the model folder.
bool SetPrompt(const std::string& text) override;
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSONNXSAM3();
private:
std::string _modelFilePath;
bool _modelLoadValid = false;
bool _fp16 = false;
// ONNX Runtime engine
std::unique_ptr<ONNXSAM3> m_engine;
// Tokenizer (loaded from merges.txt)
std::unique_ptr<ANSCLIPTokenizer> m_tokenizer;
// Segmentation threshold (applied after sigmoid)
float m_segThreshold = 0.5f;
// Token length (read from engine after load)
int m_tokenLength = 32;
// Prompt state
bool m_promptSet = false;
// Internal: create engine from model folder (3-session architecture)
bool InitEngine(const std::string& modelFolder);
};
}
#endif
File diff suppressed because it is too large Load Diff
+84
View File
@@ -0,0 +1,84 @@
#ifndef ANSONNXSEG_H
#define ANSONNXSEG_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
namespace ANSCENTER {
class ANSENGINE_API ANSONNXSEG :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSONNXSEG();
int getInstanceId() const { return instanceId_; }
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
size_t vectorProduct(const std::vector<int64_t>& vector);
void letterBox(const cv::Mat& image,
cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool auto_ = true,
bool scaleFill = false,
bool scaleUp = true,
int stride = 32);
BoundingBox scaleCoords(const cv::Size& letterboxShape,
const BoundingBox& coords,
const cv::Size& originalShape,
bool p_Clip = true);
std::vector<cv::Scalar> generateColors(const std::vector<std::string>& classNames, int seed = 42);
cv::Mat sigmoid(const cv::Mat& src);
void NMSBoxes(const std::vector<BoundingBox>& boxes,
const std::vector<float>& scores,
float scoreThreshold,
float nmsThreshold,
std::vector<int>& indices);
void drawSegmentations(cv::Mat& image,
const std::vector<Object>& results,
float maskAlpha = 0.5f) const;
void drawSegmentationsAndBoxes(cv::Mat& image,
const std::vector<Object>& results,
float maskAlpha) const;
const std::vector<cv::Scalar>& getClassColors() const { return classColors; }
void warmupModel();
bool Init(const std::string& modelPath, bool useGPU=true, int deviceId = 0);
cv::Mat preprocess(const cv::Mat& image,float*& blobPtr,std::vector<int64_t>& inputTensorShape);
std::vector<Object> postprocess(const cv::Size& origSize,const cv::Size& letterboxSize,
const std::vector<Ort::Value>& outputs, const std::string& camera_id);
std::vector<Object> segment(const cv::Mat& image, const std::string& camera_id);
std::vector<cv::Point2f> maskToPolygon(const cv::Mat& binaryMask,
const cv::Rect& boundingBox,
float simplificationEpsilon = 2.0f,
int minContourArea = 10);
private:
static std::atomic<int> instanceCounter_; // Thread-safe counter
int instanceId_;
int deviceId_ = 0;
Ort::Env env{ nullptr }; // ONNX Runtime environment
Ort::SessionOptions sessionOptions{ nullptr }; // Session options for ONNX Runtime
Ort::Session session{ nullptr }; // ONNX Runtime session for running inference
bool isDynamicInputShape{}; // Flag indicating if input shape is dynamic
cv::Size inputImageShape; // Expected input image shape for the model
// Vectors to hold allocated input and output node names
std::vector<Ort::AllocatedStringPtr> inputNameAllocs;
std::vector<const char*> inputNames;
std::vector<Ort::AllocatedStringPtr> outputNameAllocs;
std::vector<const char*> outputNames;
size_t numInputNodes, numOutputNodes; // Number of input and output nodes in the model
std::vector<cv::Scalar> classColors;
float m_imgWidth = 0;
float m_imgHeight = 0;
};
}
#endif
#pragma once
File diff suppressed because it is too large Load Diff
+229
View File
@@ -0,0 +1,229 @@
#ifndef ANSONNXYOLO_H
#define ANSONNXYOLO_H
#pragma once
#include "ANSEngineCommon.h"
#include "ONNXEngine.h"
namespace ANSCENTER {
// ====================================================================
// ONNXYOLO — Ultralytics YOLO inference via ONNX Runtime
//
// Compatible with ALL Ultralytics YOLO tasks and versions
// (v8, v9, v10, v11, v26+) by auto-detecting the task type
// from output tensor shapes at inference time.
//
// Supported tasks:
// - Detection : [B,300,6] end2end or [B, nc+4, N] legacy
// - OBB : [B,300,7] end2end or [B, nc+5, N] legacy
// - Segmentation: [B,300,38]+protos or [B, nc+36, N]+protos
// - Pose : [B,300,6+nk*3] or [B, nc+4+nk*3, N] legacy
// - Classification: [B, nc]
//
// Preprocessing follows the exact Ultralytics LetterBox transform:
// - center=True, stride=32, pad_value=114
// - Ultralytics-compatible -0.1/+0.1 rounding for deterministic padding
// - BGR→RGB, /255.0 normalisation, HWC→CHW
// ====================================================================
class ONNXENGINE_API ONNXYOLO : public BasicOrtHandler
{
public:
explicit ONNXYOLO(const std::string& _onnx_path,
unsigned int _num_threads = 1);
explicit ONNXYOLO(const std::string& _onnx_path,
EngineType engineType,
unsigned int _num_threads = 1);
~ONNXYOLO() override = default;
/// Run inference on a single image.
/// Auto-detects the task type (detect/segment/obb/pose/classify)
/// from the ONNX model's output tensor shapes.
std::vector<Object> detect(const cv::Mat& image,
const std::vector<std::string>& classNames,
float confThreshold = 0.25f,
float iouThreshold = 0.45f,
int numKPS = 0);
/// True after detect() if the last inference was classification.
bool lastWasClassification = false;
/// Run batched inference on multiple images in a single ONNX session call.
/// Falls back to sequential detect() if the model has fixed batch=1.
std::vector<std::vector<Object>> detectBatch(
const std::vector<cv::Mat>& images,
const std::vector<std::string>& classNames,
float confThreshold = 0.25f,
float iouThreshold = 0.45f,
int numKPS = 0);
/// True after detectBatch() if the batch was classification.
bool lastBatchWasClassification = false;
/// Override the input image shape for dynamic-input models.
/// Call after construction when the model config specifies a
/// different resolution than the default 640x640.
void setInputShape(int width, int height) {
inputImageShape = cv::Size(width, height);
}
/// True if the ONNX model has dynamic spatial dimensions.
bool hasDynamicInputShape() const { return isDynamicInputShape; }
private:
Ort::Value transform(const cv::Mat& mat) override;
Ort::Value transformBatch(const std::vector<cv::Mat>& images) override;
// ── Ultralytics-compatible letterbox ────────────────────────────
void letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool scaleUp = true,
int stride = 32);
// ── Detection postprocess ───────────────────────────────────────
std::vector<Object> postprocessEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── OBB postprocess ─────────────────────────────────────────────
std::vector<Object> postprocessOBBEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessOBBLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── Segmentation postprocess ────────────────────────────────────
std::vector<Object> postprocessSegEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames, float confThreshold);
std::vector<Object> postprocessSegLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet = 300);
// ── Pose postprocess ────────────────────────────────────────────
std::vector<Object> postprocessPoseEndToEnd(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, int numKPS);
std::vector<Object> postprocessPoseLegacy(
const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int numKPS, int maxDet = 300);
// ── Classification postprocess ──────────────────────────────────
std::vector<Object> postprocessClassify(
const std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
const cv::Size& imageSize);
// ── OBB NMS helpers (Prob-IoU based) ────────────────────────────
struct OrientedBox {
float x, y, width, height, angle;
};
static void getCovarianceComponents(const OrientedBox& box,
float& out1, float& out2, float& out3);
static std::vector<std::vector<float>> batchProbiou(
const std::vector<OrientedBox>& obb1,
const std::vector<OrientedBox>& obb2, float eps = 1e-7f);
static std::vector<int> nmsRotatedImpl(
const std::vector<OrientedBox>& sortedBoxes, float iouThreshold);
static std::vector<int> nmsRotated(
const std::vector<OrientedBox>& boxes,
const std::vector<float>& scores, float iouThreshold);
static std::vector<cv::Point2f> OBBToPoints(const OrientedBox& obb);
// ── Batch output slicing helper ────────────────────────────────
static Ort::Value sliceBatchOutput(
const Ort::Value& batchTensor,
int64_t batchIndex,
const std::vector<int64_t>& fullShape,
Ort::MemoryInfo& memInfo);
// Cached model input shape
cv::Size inputImageShape;
bool isDynamicInputShape{ false };
};
// ====================================================================
// ANSONNXYOLO — ANSODBase wrapper for Ultralytics YOLO ONNX
//
// Compatible with all Ultralytics YOLO tasks and versions.
// Implements all required ANSODBase interfaces.
// ====================================================================
class ANSENGINE_API ANSONNXYOLO : public ANSODBase {
public:
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath,
const std::string& modelZipPassword,
std::string& labelMap) override;
bool LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword) override;
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap) override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override;
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs,
const std::string& camera_id) override;
bool Destroy();
~ANSONNXYOLO();
private:
std::string _modelFilePath;
bool _modelLoadValid{ false };
// Filter thresholds
float PROBABILITY_THRESHOLD{ 0.25f };
float NMS_THRESHOLD{ 0.45f };
int TOP_K{ 300 };
// Pose estimation
int NUM_KPS{ 0 };
float KPS_THRESHOLD{ 0.5f };
// ONNX Runtime inference engine
std::unique_ptr<ONNXYOLO> m_ortEngine;
// Internal detection pipeline
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id);
// Internal batch detection pipeline
std::vector<std::vector<Object>> DetectObjectsBatch(
const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
// Initialise ORT engine from the resolved model path
bool InitOrtEngine();
};
}
#endif
+518
View File
@@ -0,0 +1,518 @@
#include "ANSOPENVINOCL.h"
#include "Utility.h"
namespace ANSCENTER
{
bool OPENVINOCL::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("OPENVINOCL::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("OPENVINOCL::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("OPENVINOCL::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool OPENVINOCL::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("OPENVINOCL::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("OPENVINOCL::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("OPENVINOCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("OPENVINOCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool OPENVINOCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".xml";
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, modelFullName);//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, className);
this->_logger.LogDebug("OPENVINOCL::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("OPENVINOCL::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("OPENVINOCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("OPENVINOCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
cv::Mat OPENVINOCL::PreProcessing(const cv::Mat& source) {
if (source.empty()) {
std::cerr << "Error: Empty image provided to PreProcessing" << std::endl;
return cv::Mat();
}
// Convert grayscale to 3-channel BGR
cv::Mat processedImage;
if (source.channels() == 1) {
cv::cvtColor(source, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = source;
}
int col = processedImage.cols;
int row = processedImage.rows;
int maxSize = std::max(col, row);
// Create a square padded image with a black background
cv::Mat result = cv::Mat::zeros(maxSize, maxSize, CV_8UC3);
// Copy the original image to the top-left corner of the square matrix
processedImage.copyTo(result(cv::Rect(0, 0, col, row)));
return result;
}
bool OPENVINOCL::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
std::string openVINOVersion = ov::get_openvino_version().buildNumber;
//this->_logger.LogDebug("OPENVINOCL::Initialize. OpenVINO version", openVINOVersion, __FILE__, __LINE__);
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("OPENVINOCL::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("OPENVINOCL::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("OPENVINOCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("OPENVINOCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOCL::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> OPENVINOCL::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> OPENVINOCL::RunInference(const cv::Mat& input,const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
// Early validation
if (!_licenseValid) {
_logger.LogError("OPENVINOCL::RunInference", "Invalid License",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("OPENVINOCL::RunInference", "Model is not initialized",
__FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 20 || input.rows < 20) {
_logger.LogError("OPENVINOCL::RunInference", "Input image is invalid",
__FILE__, __LINE__);
return {};
}
try {
// Step 1: Resize directly to target size (no clone needed!)
cv::Mat resized;
cv::resize(input, resized, cv::Size(224, 224), 0, 0, cv::INTER_LINEAR);
// Step 2: Preprocessing
cv::Mat letterbox_img = PreProcessing(resized);
// Step 3: Create blob
cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0,
cv::Size(224, 224), cv::Scalar(), true);
// Step 4: Set input tensor
auto input_port = compiled_model_.input();
ov::Tensor input_tensor(input_port.get_element_type(),
input_port.get_shape(), blob.ptr(0));
inference_request_.set_input_tensor(input_tensor);
// Step 5: Run inference
inference_request_.start_async();
inference_request_.wait();
// Step 6: Get output
auto output = inference_request_.get_output_tensor(0);
auto output_shape = output.get_shape();
if (output_shape.size() < 2) {
_logger.LogError("OPENVINOCL::RunInference", "Invalid output shape",
__FILE__, __LINE__);
return {};
}
// Step 7: Find max class (no vector copy needed!)
const float* output_buffer = output.data<const float>();
const size_t num_classes = output_shape[1];
auto max_ptr = std::max_element(output_buffer, output_buffer + num_classes);
const int class_id = static_cast<int>(std::distance(output_buffer, max_ptr));
const float score = *max_ptr;
// Step 8: Create result object
Object clsResult;
clsResult.classId = class_id;
clsResult.confidence = score;
clsResult.cameraId = camera_id;
// Set class name
const int classNameSize = static_cast<int>(_classes.size());
if (!_classes.empty()) {
clsResult.className = (class_id < classNameSize)
? _classes[class_id]
: _classes.back();
}
else {
clsResult.className = "Unknown";
}
// Set bounding box (full image for classification)
clsResult.box = cv::Rect(0, 0, input.cols, input.rows);
// Set polygon
clsResult.polygon = ANSUtilityHelper::RectToNormalizedPolygon(
clsResult.box, input.cols, input.rows
);
// Return single-element vector
return { std::move(clsResult) };
}
catch (const std::exception& e) {
_logger.LogFatal("OPENVINOCL::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
catch (...) {
_logger.LogFatal("OPENVINOCL::RunInference", "Unknown exception",
__FILE__, __LINE__);
return {};
}
}
OPENVINOCL::~OPENVINOCL() {
try {
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("OPENVINOCL::~OPENVINOCL", "Failed to delete OpenVINO Models", __FILE__, __LINE__);
}
}
}
catch (std::exception& e) {
this->_logger.LogError("OPENVINOCL::~OPENVINOCL()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
}
}
bool OPENVINOCL::Destroy() {
try {
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("OPENVINOCL::Destroy", "Failed to delete OpenVINO Models", __FILE__, __LINE__);
}
}
return true;
}
catch (std::exception& e) {
this->_logger.LogError("OPENVINOCL::Destroy()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
return false;
}
}
//private
void OPENVINOCL::InitialModel() {
try {
// Step 1: Initialize OpenVINO Runtime Core
ov::Core core;
// Step 2: Load Model
// Get available devices
std::vector<std::string> available_devices = core.get_available_devices();
bool device_found = false;
// Search for NPU
auto it = std::find(available_devices.begin(), available_devices.end(), "NPU");
if (it != available_devices.end()) {
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "AUTO:NPU,GPU");
device_found = true;
//this->_logger.LogDebug("OPENVINOCL::InitialModel", "Using NPU for inference.", __FILE__, __LINE__);
}
// If NPU not found, search for GPU
if (!device_found) {
it = std::find(available_devices.begin(), available_devices.end(), "GPU");
if (it != available_devices.end()) {
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "GPU");
device_found = true;
//this->_logger.LogDebug("OPENVINOCL::InitialModel", "Using GPU for inference.", __FILE__, __LINE__);
}
}
// If GPU not found, search for GPU.0
if (!device_found) {
it = std::find(available_devices.begin(), available_devices.end(), "GPU.0");
if (it != available_devices.end()) {
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "GPU");
device_found = true;
//this->_logger.LogDebug("OPENVINOCL::InitialModel", "Using GPU for inference.", __FILE__, __LINE__);
}
}
// If neither NPU nor GPU found, default to CPU
if (!device_found) {
core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "CPU");
//this->_logger.LogDebug("OPENVINOCL::InitialModel", "Using CPU for inference.", __FILE__, __LINE__);
}
// Step 3: Create Inference Request
inference_request_ = compiled_model_.create_infer_request();
}
catch (const std::exception& e) {
this->_logger.LogFatal("OPENVINOCL::InitialModel", e.what(), __FILE__, __LINE__);
}
}
}
//std::vector<Object> OPENVINOCL::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> outputs;
// if (!_licenseValid) {
// this->_logger.LogError("OPENVINOCL::RunInference", "Invalid License", __FILE__, __LINE__);
// return outputs;
// }
// if (!_isInitialized) {
// this->_logger.LogError("OPENVINOCL::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return outputs;
// }
// try {
// // Step 0: Prepare input
// if (input.empty()) {
// this->_logger.LogError("OPENVINOCL::RunInference", "Input image is empty", __FILE__, __LINE__);
// return outputs;
// }
// if ((input.cols < 20) || (input.rows < 20)) return outputs;
// cv::Mat img = input.clone();
// cv::resize(img, img, cv::Size(224, 224));
// cv::Mat letterbox_img = PreProcessing(img);
// cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0, cv::Size(224, 224), cv::Scalar(), true);
// // Step 1: Feed blob to the network
// auto input_port = compiled_model_.input();
// ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blob.ptr(0));
// inference_request_.set_input_tensor(input_tensor);
// // Step 3: Start inference
// inference_request_.start_async();
// inference_request_.wait();
// // Step 4: Get output
// auto output = inference_request_.get_output_tensor(0);
// auto output_shape = output.get_shape();
// if (output_shape.size() < 2) {
// this->_logger.LogError("OPENVINOCL::RunInference", "Invalid output shape", __FILE__, __LINE__);
// return outputs;
// }
// // Step 5: Post-processing
// const float* output_buffer = output.data<const float>();
// std::vector<float> result(output_buffer, output_buffer + output_shape[1]);
// auto max_idx = std::max_element(result.begin(), result.end());
// int class_id = static_cast<int>(std::distance(result.begin(), max_idx));
// float score = *max_idx;
// int classNameSize = static_cast<int>(_classes.size());
// Object clsResult;
// clsResult.classId = class_id;
// //clsResult.className = (class_id < _classes.size()) ? _classes[class_id] : "";
// if (!_classes.empty()) {
// if (clsResult.classId < classNameSize) {
// clsResult.className = _classes[clsResult.classId];
// }
// else {
// clsResult.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
// }
// }
// else {
// clsResult.className = "Unknown"; // Fallback if _classes is empty
// }
// clsResult.confidence = score;
// if (input.cols > 20 && input.rows > 20) {
// clsResult.box = cv::Rect(10, 10, input.cols - 20, input.rows - 20);
// }
// else {
// clsResult.box = cv::Rect(0, 0, input.cols, input.rows);
// }
// clsResult.polygon = ANSUtilityHelper::RectToNormalizedPolygon(clsResult.box, input.cols, input.rows);
// clsResult.cameraId = camera_id;
// outputs.push_back(clsResult);
// return outputs;
// }
// catch (const std::exception& e) {
// this->_logger.LogFatal("OPENVINOCL::RunInference", e.what(), __FILE__, __LINE__);
// }
// catch (...) {
// this->_logger.LogFatal("OPENVINOCL::RunInference", "Unknown exception occurred", __FILE__, __LINE__);
// }
// return outputs;
//}
+32
View File
@@ -0,0 +1,32 @@
#ifndef ANSOPENVINOCL_H
#define ANSOPENVINOCL_H
#pragma once
#include "ANSEngineCommon.h"
#include<openvino/openvino.hpp>
namespace ANSCENTER {
class ANSENGINE_API OPENVINOCL :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~OPENVINOCL();
private:
std::string _modelFilePath;
private:
void InitialModel();
cv::Mat PreProcessing(const cv::Mat& source);
cv::Mat resized_frame_;
cv::Point2f factor_;
cv::Size2f model_input_shape_;
cv::Size model_output_shape_;
ov::Tensor input_tensor_;
ov::InferRequest inference_request_;
ov::CompiledModel compiled_model_;
};
}
#endif
+608
View File
@@ -0,0 +1,608 @@
#include "ANSOPENVINOOD.h"
#include "Utility.h"
namespace ANSCENTER
{
bool OPENVINOOD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("OPENVINOOD::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("OPENVINOOD::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("OPENVINOOD::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool OPENVINOOD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("OPENVINOOD::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("OPENVINOOD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("OPENVINOOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("OPENVINOOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOOD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool OPENVINOOD::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".xml";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, modelFullName);//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, className);
this->_logger.LogDebug("OPENVINOOD::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("OPENVINOOD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("OPENVINOOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("OPENVINOOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOOD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
cv::Mat OPENVINOOD::PreProcessing(const cv::Mat& source) {
try {
if (source.empty()) {
this->_logger.LogFatal("OPENVINOOD::PreProcessing", "Empty image provided", __FILE__, __LINE__);
return cv::Mat();
}
// Convert grayscale to 3-channel BGR if needed
cv::Mat processedImage;
if (source.channels() == 1) {
cv::cvtColor(source, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = source;
}
int col = processedImage.cols;
int row = processedImage.rows;
int maxSize = std::max(col, row);
// Create a square padded image with a black background
cv::Mat result = cv::Mat::zeros(maxSize, maxSize, CV_8UC3);
// Copy the original image to the top-left corner of the square matrix
processedImage.copyTo(result(cv::Rect(0, 0, col, row)));
return result;
}
catch (const std::exception& e) {
this->_logger.LogFatal("OPENVINOOD::PreProcessing", e.what(), __FILE__, __LINE__);
return cv::Mat();
}
}
bool OPENVINOOD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
model_input_shape_ = cv::Size2f(_modelConfig.inpWidth, _modelConfig.inpHeight);
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("OPENVINOOD::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("OPENVINOOD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("OPENVINOOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("OPENVINOOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOOD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> OPENVINOOD::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> OPENVINOOD::RunInference(const cv::Mat& input, const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
// Early validation
if (!_licenseValid) {
_logger.LogError("OPENVINOOD::RunInference", "Invalid License",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("OPENVINOOD::RunInference", "Model is not initialized",
__FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
_logger.LogError("OPENVINOOD::RunInference", "Invalid input image",
__FILE__, __LINE__);
return {};
}
try {
// Step 1: Preprocessing
cv::Mat letterbox_img = PreProcessing(input);
if (letterbox_img.empty()) {
_logger.LogError("OPENVINOOD::RunInference", "PreProcessing failed",
__FILE__, __LINE__);
return {};
}
// Step 2: Create blob
constexpr int imageSize = 640;
const float scale = static_cast<float>(letterbox_img.rows) / imageSize;
cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0,
cv::Size(imageSize, imageSize),
cv::Scalar(), true);
// Step 3: Run inference
auto input_port = compiled_model_.input();
ov::Tensor input_tensor(input_port.get_element_type(),
input_port.get_shape(), blob.ptr(0));
inference_request_.set_input_tensor(input_tensor);
inference_request_.infer();
// Step 4: Get output
auto output = inference_request_.get_output_tensor(0);
auto output_shape = output.get_shape();
if (output_shape.size() != 3) {
_logger.LogError("OPENVINOOD::RunInference", "Unexpected output shape",
__FILE__, __LINE__);
return {};
}
const int rows = static_cast<int>(output_shape[2]); // 8400
const int dimensions = static_cast<int>(output_shape[1]); // 84
float* data = output.data<float>();
// Step 5: Parse detections (avoid transpose!)
std::vector<int> class_ids;
std::vector<float> class_scores;
std::vector<cv::Rect> boxes;
// Pre-allocate for efficiency
class_ids.reserve(rows / 10); // Estimate ~10% pass threshold
class_scores.reserve(rows / 10);
boxes.reserve(rows / 10);
const cv::Rect imageBounds(0, 0, input.cols, input.rows);
// Process detections without transpose
for (int i = 0; i < rows; i++) {
// Get class scores starting at index 4
float max_score = -1.0f;
int max_class_id = 0;
for (int j = 4; j < dimensions; j++) {
const float score = data[j * rows + i]; // Column-major access
if (score > max_score) {
max_score = score;
max_class_id = j - 4;
}
}
if (max_score > _modelConfig.detectionScoreThreshold) {
// Extract box coordinates
const float cx = data[0 * rows + i];
const float cy = data[1 * rows + i];
const float w = data[2 * rows + i];
const float h = data[3 * rows + i];
// Convert to pixel coordinates and clamp
cv::Rect box(
static_cast<int>((cx - 0.5f * w) * scale),
static_cast<int>((cy - 0.5f * h) * scale),
static_cast<int>(w * scale),
static_cast<int>(h * scale)
);
box &= imageBounds; // Clamp to image bounds
if (box.area() > 0) {
class_ids.push_back(max_class_id);
class_scores.push_back(max_score);
boxes.push_back(box);
}
}
}
// Step 6: Apply NMS
if (boxes.empty()) {
return {};
}
std::vector<int> indices;
cv::dnn::NMSBoxes(boxes, class_scores,
_modelConfig.detectionScoreThreshold,
_modelConfig.modelMNSThreshold, indices);
// Step 7: Build output objects
std::vector<Object> outputs;
outputs.reserve(indices.size());
const int classNameSize = static_cast<int>(_classes.size());
for (int idx : indices) {
Object result;
result.classId = class_ids[idx];
result.confidence = class_scores[idx];
result.box = boxes[idx];
result.cameraId = camera_id;
// Set class name
if (!_classes.empty()) {
result.className = (result.classId < classNameSize)
? _classes[result.classId]
: _classes.back();
}
else {
result.className = "Unknown";
}
// Set polygon
result.polygon = ANSUtilityHelper::RectToNormalizedPolygon(
result.box, input.cols, input.rows
);
outputs.push_back(std::move(result));
}
if (_trackerEnabled) {
outputs = ApplyTracking(outputs, camera_id);
if (_stabilizationEnabled) outputs = StabilizeDetections(outputs, camera_id);
}
return outputs;
}
catch (const std::exception& e) {
_logger.LogFatal("OPENVINOOD::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
catch (...) {
_logger.LogFatal("OPENVINOOD::RunInference", "Unknown error",
__FILE__, __LINE__);
return {};
}
}
OPENVINOOD::~OPENVINOOD() {
try {
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("OPENVINOOD::~OPENVINOOD", "Failed to delete OpenVINO Models", __FILE__, __LINE__);
}
}
}
catch (std::exception& e) {
this->_logger.LogError("OPENVINOOD::~OPENVINOOD()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
}
}
bool OPENVINOOD::Destroy() {
try {
if (FolderExist(_modelFolder)) {
DeleteFolder(_modelFolder);
}
return true;
}
catch (std::exception& e) {
this->_logger.LogError("OPENVINOOD::Destroy()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
return false;
}
}
//private
void OPENVINOOD::InitialModel() {
try {
// Step 1: Initialize OpenVINO Runtime Core
ov::Core core;
// Step 2: Get Available Devices and Log
std::vector<std::string> available_devices = core.get_available_devices();
// Define device priority: NPU > GPU > CPU
std::vector<std::string> priority_devices = { "NPU", "GPU" };
bool device_found = false;
// Iterate over prioritized devices
for (const auto& device : priority_devices) {
if (std::find(available_devices.begin(), available_devices.end(), device) != available_devices.end()) {
if (device == "NPU") {
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
compiled_model_ = core.compile_model(_modelFilePath, "MULTI:NPU, GPU");
}
else {
// Configure and compile for individual device
//core.set_property(device, ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
compiled_model_ = core.compile_model(_modelFilePath, device);
}
device_found = true;
break;
}
}
// Fallback: Default to CPU if no devices found
if (!device_found) {
//core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
compiled_model_ = core.compile_model(_modelFilePath, "CPU");
}
// Step 3: Create Inference Request
inference_request_ = compiled_model_.create_infer_request();
}
catch (const std::exception& e) {
// Log any errors
this->_logger.LogFatal("OPENVINOOD::InitialModel", e.what(), __FILE__, __LINE__);
}
}
}
//std::vector<Object> OPENVINOOD::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> outputs;
// if (!_licenseValid) {
// _logger.LogError("OPENVINOOD::RunInference", "Invalid License", __FILE__, __LINE__);
// return outputs;
// }
// if (!_isInitialized) {
// _logger.LogError("OPENVINOOD::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return outputs;
// }
// if (input.empty()) {
// _logger.LogError("OPENVINOOD::RunInference", "Input image is empty", __FILE__, __LINE__);
// return outputs;
// }
// try {
// // Step 0: Prepare input
// if (input.empty()) return outputs;
// if ((input.cols < 10) || (input.rows < 10)) return outputs;
// cv::Mat letterbox_img = PreProcessing(input);
// if (letterbox_img.empty()) {
// _logger.LogError("OPENVINOOD::RunInference", "PreProcessing failed", __FILE__, __LINE__);
// return outputs;
// }
// int imageSize = 640;
// int maxImageSize = std::max(letterbox_img.cols, letterbox_img.rows);
// //if (maxImageSize < imageSize)imageSize = maxImageSize;
// float scale = static_cast<float>(letterbox_img.rows) / imageSize;
// cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0,
// cv::Size(imageSize, imageSize),
// cv::Scalar(), true);
// // Step 1: Feed blob to the network
// auto input_port = compiled_model_.input();
// ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blob.ptr(0));
// inference_request_.set_input_tensor(input_tensor);
// inference_request_.infer();
// // Step 4: Get output
// auto output = inference_request_.get_output_tensor(0);
// auto output_shape = output.get_shape();
// if (output_shape.size() != 3) {
// _logger.LogError("OPENVINOOD::RunInference", "Unexpected output shape", __FILE__, __LINE__);
// return outputs;
// }
// int rows = output_shape[2]; // 8400
// int dimensions = output_shape[1]; // 84: box[cx, cy, w, h]+80 class scores
// float* data = output.data<float>();
// cv::Mat output_buffer(dimensions, rows, CV_32F, data);
// transpose(output_buffer, output_buffer); // [8400, 84]
// std::vector<int> class_ids;
// std::vector<float> class_scores;
// std::vector<cv::Rect> boxes;
// // Step 5: Post-processing
// for (int i = 0; i < output_buffer.rows; i++) {
// cv::Mat classes_scores = output_buffer.row(i).colRange(4, dimensions);
// cv::Point class_id;
// double max_class_score;
// minMaxLoc(classes_scores, nullptr, &max_class_score, nullptr, &class_id);
// if (max_class_score > _modelConfig.detectionScoreThreshold) {
// class_scores.push_back(static_cast<float>(max_class_score));
// class_ids.push_back(class_id.x);
// float cx = output_buffer.at<float>(i, 0);
// float cy = output_buffer.at<float>(i, 1);
// float w = output_buffer.at<float>(i, 2);
// float h = output_buffer.at<float>(i, 3);
// int left = static_cast<int>((cx - 0.5f * w) * scale);
// int top = static_cast<int>((cy - 0.5f * h) * scale);
// int width = static_cast<int>(w * scale);
// int height = static_cast<int>(h * scale);
// left = std::max(0, left);
// top = std::max(0, top);
// width = std::min(input.cols - left, width);
// height = std::min(input.rows - top, height);
// boxes.emplace_back(left, top, width, height);
// }
// }
// // NMS
// int classNameSize = static_cast<int>(_classes.size());
// std::vector<int> indices;
// cv::dnn::NMSBoxes(boxes, class_scores, _modelConfig.detectionScoreThreshold, _modelConfig.modelMNSThreshold, indices);
// for (int id : indices) {
// if (class_scores[id] >= _modelConfig.detectionScoreThreshold) {
// Object result;
// int class_id = class_ids[id];
// result.classId = class_id;
// if (!_classes.empty()) {
// if (result.classId < classNameSize) {
// result.className = _classes[result.classId];
// }
// else {
// result.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
// }
// }
// else {
// result.className = "Unknown"; // Fallback if _classes is empty
// }
// result.confidence = class_scores[id];
// result.box = boxes[id];
// result.polygon = ANSUtilityHelper::RectToNormalizedPolygon(result.box, input.cols, input.rows);
// result.cameraId = camera_id;
// outputs.push_back(result);
// }
// }
// }
// catch (const std::exception& e) {
// _logger.LogFatal("OPENVINOOD::RunInference", e.what(), __FILE__, __LINE__);
// }
// catch (...) {
// _logger.LogFatal("OPENVINOOD::RunInference", "Unknown error occurred", __FILE__, __LINE__);
// }
// return outputs; //
//}
+32
View File
@@ -0,0 +1,32 @@
#ifndef ANSOPENVINOOD_H
#define ANSOPENVINOOD_H
#pragma once
#include "ANSEngineCommon.h"
#include<openvino/openvino.hpp>
namespace ANSCENTER {
class ANSENGINE_API OPENVINOOD :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~OPENVINOOD();
private:
std::string _modelFilePath;
private:
void InitialModel();
cv::Mat PreProcessing(const cv::Mat& source);
cv::Mat resized_frame_;
cv::Point2f factor_;
cv::Size2f model_input_shape_;
cv::Size model_output_shape_;
ov::Tensor input_tensor_;
ov::InferRequest inference_request_;
ov::CompiledModel compiled_model_;
};
}
#endif
+277
View File
@@ -0,0 +1,277 @@
#include "ANSOVFBFaceDetector.h"
#include <models/detection_model_faceboxes.h>
#include <pipelines/metadata.h>
#include <models/input_data.h>
#include <models/results.h>
#include "Utility.h"
namespace ANSCENTER {
bool ANSOVFBFD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
bool result = ANSFDBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
labelMap = "Face";
// We do not need to check for the license
_licenseValid = true;
if (!_licenseValid) return false;
try {
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::FACEDETECT;
_modelConfig.detectionType = DetectionType::FACEDETECTOR;
// We need to get the modelfolder from here
std::string xmlfile = CreateFilePath(_modelFolder, "facebox.xml");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSOVFBFD::Initialize. Loading ansovfb weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVFBFD::Initialize. Model ansovfb.xml file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::string landmarkModel = CreateFilePath(_modelFolder, "landmarks.xml");
if (std::filesystem::exists(landmarkModel)) {
_landmarkModelFilePath = landmarkModel;
this->_logger.LogDebug("ANSOVFBFD::Initialize. Loading landmarks weight", _landmarkModelFilePath, __FILE__, __LINE__);
ov::Core lmcore;
CnnConfig landmarks_config(_landmarkModelFilePath, "Facial Landmarks Regression");
landmarks_config.m_deviceName = "GPU";
landmarks_config.m_max_batch_size = 1;
landmarks_config.m_core = lmcore;
this->_faceLandmark = std::make_unique<VectorCNN>(landmarks_config);
if (_faceLandmark == nullptr) {
this->_logger.LogFatal("ANSOVFBFD::Initialize", "Failed to initialize face recognizer model", __FILE__, __LINE__);
}
}
else {
this->_logger.LogError("ANSOVFBFD::Initialize. Model landmarks.xml file is not exist", _landmarkModelFilePath, __FILE__, __LINE__);
}
std::vector<std::string> labels;
labels.push_back(labelMap);
std::unique_ptr<ModelBase> _faceDetector;
_faceDetector.reset(new ModelFaceBoxes(_modelFilePath,
static_cast<float>(0.5),
false,
static_cast<float>(0.5),
""));
_faceDetector->setInputsPreprocessing(false, "", "");
ov::Core core;
this->_faceDectectorPipeline = std::make_unique<AsyncPipeline>(std::move(_faceDetector), ConfigFactory::getUserConfig("GPU", 0, "", 0), core);
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFBFD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVFBFD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
// We need to get the _modelFolder
bool result = ANSFDBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// We need to get the modelfolder from here
std::string xmlfile = CreateFilePath(_modelFolder, "ansovfb.xml");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSOVFBFD::LoadModel. Loading ansovfb weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVFBFD::LoadModel. Model ansovfb.xml file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFBFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVFBFD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (!FileExist(_modelFilePath)) {
optimizedModelFolder = "";
return false;
}
return true;
}
std::vector<Object> ANSOVFBFD::RunInference(const cv::Mat& input) {
std::vector<Object> output;
output.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSOVFBFD::RunInference", "Invalid license", __FILE__, __LINE__);
return output;
}
if (!_isInitialized) {
this->_logger.LogError("ANSOVFBFD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return output;
}
try {
bool croppedFace = false; // Check if the image is cropped face image
cv::Mat im = input.clone();
//std::cout << "Width x Height=" << input.size[0] << "x" << input.size[1] << std::endl;
// We know that the image sizes <=300 px, it is likely that image is cropped for face only
if ((input.size[0] <= 300) || (input.size[1] <= 300)) croppedFace = true;
if (croppedFace) cv::copyMakeBorder(input, im, 200, 200, 200, 200, cv::BORDER_REPLICATE);
;
auto startTime = std::chrono::steady_clock::now();
int64_t frameNum = _faceDectectorPipeline->submitData(ImageInputData(im), std::make_shared<ImageMetaData>(im, startTime));
_faceDectectorPipeline->waitForTotalCompletion();
std::unique_ptr<ResultBase> result = _faceDectectorPipeline->getResult();
auto outputImg = result->metaData->asRef<ImageMetaData>().img;
if (outputImg.empty()) {
throw std::invalid_argument("Renderer: image provided in metadata is empty");
}
for (auto& obj : result->asRef<DetectionResult>().objects) {
Object result;
std::vector<cv::Mat> face_rois;
std::vector<cv::Mat> landmarks;
float confidence = obj.confidence;
if (confidence >= _modelConfig.detectionScoreThreshold)
{
int x_min = obj.x;
int y_min = obj.y;
int x_max = obj.x + obj.width;
int y_max = obj.y + obj.height;
result.classId = obj.labelID;
result.className = obj.label;
result.confidence = confidence;
result.box.x = x_min;
result.box.y = y_min;
if (croppedFace) {
if (x_min <= 200) x_min = 200;
if (y_min <= 200) y_min = 200;
result.box.x = x_min - 200;
result.box.y = y_min - 200;
}
result.box.width = x_max - x_min;
result.box.height = y_max - y_min;
if (_faceLandmark != nullptr) {
landmarks.clear();
face_rois.clear();
cv::Mat face_roi = ANSUtilityHelper::GetCroppedFace(im, x_min, y_min, x_max, y_max);
face_rois.push_back(face_roi);
_faceLandmark->Compute(face_rois, &landmarks, cv::Size(2, 5));
ANSUtilityHelper::AlignFaces(&face_rois, &landmarks);
result.mask = face_rois[0];
face_roi.release();
}
else {
result.mask = ANSUtilityHelper::GetCroppedFace(im, x_min, y_min, x_max, y_max);
}
result.cameraId = "FBFaceCAM";
output.push_back(result);
}
}
im.release();
EnqueueDetection(output, "FBFaceCAM");
return output;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFBFD::RunInference", e.what(), __FILE__, __LINE__);
return output;
}
}
std::vector<Object> ANSOVFBFD::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::vector<Object> output;
output.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSOVFBFD::RunInference", "Invalid license", __FILE__, __LINE__);
return output;
}
if (!_isInitialized) {
this->_logger.LogError("ANSOVFBFD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return output;
}
try {
bool croppedFace = false; // Check if the image is cropped face image
cv::Mat im = input.clone();
//std::cout << "Width x Height=" << input.size[0] << "x" << input.size[1] << std::endl;
// We know that the image sizes <=300 px, it is likely that image is cropped for face only
if ((input.size[0] <= 300) || (input.size[1] <= 300)) croppedFace = true;
if (croppedFace) cv::copyMakeBorder(input, im, 200, 200, 200, 200, cv::BORDER_REPLICATE);
;
auto startTime = std::chrono::steady_clock::now();
int64_t frameNum = _faceDectectorPipeline->submitData(ImageInputData(im), std::make_shared<ImageMetaData>(im, startTime));
_faceDectectorPipeline->waitForTotalCompletion();
std::unique_ptr<ResultBase> result = _faceDectectorPipeline->getResult();
auto outputImg = result->metaData->asRef<ImageMetaData>().img;
if (outputImg.empty()) {
throw std::invalid_argument("Renderer: image provided in metadata is empty");
}
for (auto& obj : result->asRef<DetectionResult>().objects) {
Object result;
std::vector<cv::Mat> face_rois;
std::vector<cv::Mat> landmarks;
float confidence = obj.confidence;
if (confidence >= _modelConfig.detectionScoreThreshold)
{
int x_min = obj.x;
int y_min = obj.y;
int x_max = obj.x + obj.width;
int y_max = obj.y + obj.height;
result.classId = obj.labelID;
result.className = obj.label;
result.confidence = confidence;
result.box.x = x_min;
result.box.y = y_min;
if (croppedFace) {
if (x_min <= 200) x_min = 200;
if (y_min <= 200) y_min = 200;
result.box.x = x_min - 200;
result.box.y = y_min - 200;
}
result.box.width = x_max - x_min;
result.box.height = y_max - y_min;
if (_faceLandmark != nullptr) {
landmarks.clear();
face_rois.clear();
cv::Mat face_roi = ANSUtilityHelper::GetCroppedFace(im, x_min, y_min, x_max, y_max);
face_rois.push_back(face_roi);
_faceLandmark->Compute(face_rois, &landmarks, cv::Size(2, 5));
ANSUtilityHelper::AlignFaces(&face_rois, &landmarks);
result.mask = face_rois[0];
face_roi.release();
}
else {
result.mask = ANSUtilityHelper::GetCroppedFace(im, x_min, y_min, x_max, y_max);
}
result.cameraId = camera_id;
output.push_back(result);
}
}
im.release();
EnqueueDetection(output, camera_id);
return output;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFBFD::RunInference", e.what(), __FILE__, __LINE__);
return output;
}
}
ANSOVFBFD::~ANSOVFBFD() {
try {
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFBFD::Destroy", e.what(), __FILE__, __LINE__);
}
}
bool ANSOVFBFD::Destroy() {
try {
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFBFD::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
}
+30
View File
@@ -0,0 +1,30 @@
#ifndef ANSOVFBFaceDetector_H
#define ANSOVFBFaceDetector_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
#include "cnn.hpp"
#include "pipelines/async_pipeline.h"
namespace ANSCENTER {
class ANSENGINE_API ANSOVFBFD :public ANSFDBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSOVFBFD();
private:
ModelConfig _modelConfig;
std::string _modelFilePath;
std::string _landmarkModelFilePath;
std::unique_ptr<AsyncPipeline>_faceDectectorPipeline = nullptr;
std::unique_ptr<VectorCNN> _faceLandmark = nullptr;
};
}
#endif
+740
View File
@@ -0,0 +1,740 @@
#include "ANSOVFaceDetector.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h" // tl_currentGpuFrame()
#include <models/detection_model_ssd.h>
#include <pipelines/metadata.h>
#include <models/input_data.h>
#include <models/results.h>
#include "Utility.h"
//#define FNS_DEBUG
namespace ANSCENTER {
bool ANSOVFD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
bool result = ANSFDBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
labelMap = "Face";
_licenseValid = true;
std::vector<std::string> labels{ labelMap };
if (!_licenseValid) return false;
try {
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::FACEDETECT;
_modelConfig.detectionType = DetectionType::FACEDETECTOR;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_isInitialized) {
_face_detector.reset(); // Releases previously allocated memory for face detection
_isInitialized = false; // Reset initialization flag
}
std::string onnxModel = CreateFilePath(_modelFolder, "scrfd.onnx");
this->_face_detector = std::make_unique<SCRFD>(onnxModel);
_isInitialized = true;
_movementObjects.clear();
_retainDetectedFaces = 0;
return true;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSOVFD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVFD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
// We need to get the _modelFolder
bool result = ANSFDBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
std::string onnxModel = CreateFilePath(_modelFolder, "scrfd.onnx");
//this->_face_detector = std::make_unique<SCRFD>(onnxModel);
_isInitialized = true;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVFD::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName,std::string className, const std::string& modelFolder, std::string& labelMap) {
try {
bool result = ANSFDBase::LoadModelFromFolder(licenseKey, modelConfig,modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "scrfd";
}
std::string modelFullName = _modelName +".onnx";
// We need to get the modelfolder from here
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::FACEDETECT;
_modelConfig.detectionType = DetectionType::FACEDETECTOR;
_modelConfig.inpHeight = 640; // Only for Yolo model
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_isInitialized) {
_face_detector.reset(); // Releases previously allocated memory for face detection
_isInitialized = false; // Reset initialization flag
}
this->_face_detector = std::make_unique<SCRFD>(modelFullName);
_isInitialized = true;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVFD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (!FileExist(_modelFilePath)) {
optimizedModelFolder = "";
return false;
}
bool result = ANSFDBase::OptimizeModel(fp16, optimizedModelFolder);
return result;
}
std::vector<Object> ANSOVFD::RunInference(const cv::Mat& input, const std::string& camera_id, bool useDynamicImage, bool validateFace, bool facelivenessCheck) {
if (facelivenessCheck) {
std::vector<Object> rawFaceResults = Inference(input, camera_id, useDynamicImage, validateFace);
std::vector<Object> facesWithLivenessResults = ValidateLivenessFaces(input, rawFaceResults, camera_id);
return facesWithLivenessResults;
}
else {
return Inference(input, camera_id, useDynamicImage, validateFace);
}
}
std::vector<Object> ANSOVFD::RunInference(const cv::Mat& input, bool useDynamicImage, bool validateFace, bool facelivenessCheck) {
if (facelivenessCheck) {
std::vector<Object> rawFaceResults = Inference(input, "CustomCam", useDynamicImage, validateFace);
std::vector<Object> facesWithLivenessResults = ValidateLivenessFaces(input, rawFaceResults, "CustomCam");
return facesWithLivenessResults;
}
else {
return Inference(input, "CustomCam", useDynamicImage, validateFace);
}
}
std::vector<Object> ANSOVFD::Inference(const cv::Mat& input,
const std::string& camera_id,
bool useDynamicImage,
bool validateFace)
{
std::lock_guard<std::mutex> guard(_mtx);
try {
// Step 1: Validate license and initialization
if (!_licenseValid) {
_logger.LogError("ANSOVFD::RunInference", "Invalid license",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized || !_face_detector) {
_logger.LogError("ANSOVFD::RunInference",
"Model or face detector not initialized",
__FILE__, __LINE__);
return {};
}
// Step 2: Validate input image
if (input.empty() || input.cols < 5 || input.rows < 5) {
_logger.LogError("ANSOVFD::RunInference", "Invalid input image",
__FILE__, __LINE__);
return {};
}
// Step 3: Convert to BGR if needed
cv::Mat processedImage;
if (input.channels() == 1) {
cv::cvtColor(input, processedImage, cv::COLOR_GRAY2BGR);
}
else if (input.channels() == 3) {
processedImage = input; // Shallow copy - safe
}
else {
_logger.LogError("ANSOVFD::RunInference",
"Unsupported number of image channels",
__FILE__, __LINE__);
return {};
}
// Step 4: Prepare image for detection
const bool croppedFace = !useDynamicImage;
const int originalWidth = input.cols;
const int originalHeight = input.rows;
cv::Mat im;
int offsetX = 0, offsetY = 0; // Track padding offset
if (croppedFace) {
// Add padding
cv::copyMakeBorder(processedImage, im, 200, 200, 200, 200,
cv::BORDER_REPLICATE);
offsetX = 200;
offsetY = 200;
// Resize if too large
if (im.rows > 1280) {
const float aspectRatio = static_cast<float>(im.cols) / im.rows;
const int newHeight = 1280;
const int newWidth = static_cast<int>(newHeight * aspectRatio);
// Update offsets for resize
const float scale = static_cast<float>(newHeight) / im.rows;
offsetX = static_cast<int>(offsetX * scale);
offsetY = static_cast<int>(offsetY * scale);
cv::resize(im, im, cv::Size(newWidth, newHeight));
}
}
else {
im = processedImage; // Shallow copy
}
// Step 5: Run face detection
std::vector<ANSCENTER::types::BoxfWithLandmarks> detectionResults;
_face_detector->detect(im, detectionResults);
// Step 6: Process detections
std::vector<Object> output;
output.reserve(detectionResults.size()); // Pre-allocate
// Try to get full-res image from registry for higher quality face alignment.
// On Intel-only systems (no NVIDIA GPU), NV12 may be in CPU memory from
// D3D11VA decode. Convert to BGR once and use for all face alignments.
cv::Mat fullResImg;
float landmarkScaleX = 1.f, landmarkScaleY = 1.f;
if (!croppedFace && !detectionResults.empty()) {
auto* gpuData = tl_currentGpuFrame();
if (gpuData && gpuData->width > im.cols && gpuData->height > im.rows) {
if (gpuData->pixelFormat == 1000 && gpuData->yPlane) {
// BGR full-res from ANSVideoPlayer/ANSFilePlayer
fullResImg = cv::Mat(gpuData->height, gpuData->width, CV_8UC3,
gpuData->yPlane, gpuData->yLinesize).clone();
landmarkScaleX = static_cast<float>(gpuData->width) / im.cols;
landmarkScaleY = static_cast<float>(gpuData->height) / im.rows;
}
else if (gpuData->pixelFormat == 23 && !gpuData->isCudaDevicePtr &&
gpuData->yPlane && gpuData->uvPlane &&
(gpuData->width % 2) == 0 && (gpuData->height % 2) == 0) {
// CPU NV12 — convert to BGR for face alignment
try {
cv::Mat yPlane(gpuData->height, gpuData->width, CV_8UC1,
gpuData->yPlane, gpuData->yLinesize);
cv::Mat uvPlane(gpuData->height / 2, gpuData->width / 2, CV_8UC2,
gpuData->uvPlane, gpuData->uvLinesize);
cv::cvtColorTwoPlane(yPlane, uvPlane, fullResImg, cv::COLOR_YUV2BGR_NV12);
landmarkScaleX = static_cast<float>(gpuData->width) / im.cols;
landmarkScaleY = static_cast<float>(gpuData->height) / im.rows;
} catch (...) { /* NV12 conversion failed — skip full-res */ }
}
else if (gpuData->pixelFormat == 23 && gpuData->isCudaDevicePtr &&
gpuData->cpuYPlane && gpuData->cpuUvPlane &&
(gpuData->width % 2) == 0 && (gpuData->height % 2) == 0) {
// CUDA NV12 with CPU fallback planes
try {
cv::Mat yPlane(gpuData->height, gpuData->width, CV_8UC1,
gpuData->cpuYPlane, gpuData->cpuYLinesize);
cv::Mat uvPlane(gpuData->height / 2, gpuData->width / 2, CV_8UC2,
gpuData->cpuUvPlane, gpuData->cpuUvLinesize);
cv::cvtColorTwoPlane(yPlane, uvPlane, fullResImg, cv::COLOR_YUV2BGR_NV12);
landmarkScaleX = static_cast<float>(gpuData->width) / im.cols;
landmarkScaleY = static_cast<float>(gpuData->height) / im.rows;
} catch (...) { /* NV12 conversion failed — skip full-res */ }
}
}
}
for (const auto& obj : detectionResults) {
// Check confidence threshold
const float confidence = obj.box.score;
if (confidence < _modelConfig.detectionScoreThreshold) {
continue;
}
// Validate face landmarks
if (validateFace && !isValidFace(obj.landmarks.points,obj.box.rect(), 27)) {
continue;
}
Object result;
result.classId = 0;
result.className = "Face";
result.confidence = confidence;
result.cameraId = camera_id;
// Get bounding box
cv::Rect bbox = obj.box.rect();
// Process face for mask — use full-res image when available
// for higher quality 112x112 face alignment
cv::Mat alignImage;
std::vector<cv::Point2f> face_landmark_5 = obj.landmarks.points;
if (!fullResImg.empty()) {
// Scale landmarks from display-res to full-res
for (auto& pt : face_landmark_5) {
pt.x *= landmarkScaleX;
pt.y *= landmarkScaleY;
}
alignImage = fullResImg;
} else {
alignImage = im;
}
result.mask = Preprocess(alignImage, face_landmark_5, alignImage);
if (result.mask.empty()) {
_logger.LogError("ANSOVFD::RunInference", "Cannot get mask image",
__FILE__, __LINE__);
continue;
}
// Adjust coordinates back to original image space
if (croppedFace) {
// Remove padding offset
bbox.x = std::max(0, bbox.x - offsetX);
bbox.y = std::max(0, bbox.y - offsetY);
// Clamp to original image bounds
bbox.x = std::min(bbox.x, originalWidth);
bbox.y = std::min(bbox.y, originalHeight);
bbox.width = std::min(bbox.width, originalWidth - bbox.x);
bbox.height = std::min(bbox.height, originalHeight - bbox.y);
result.box = bbox;
// Adjust landmarks
result.kps.reserve(obj.landmarks.points.size() * 2);
for (const auto& pt : obj.landmarks.points) {
result.kps.push_back(std::max(0.0f, pt.x - offsetX));
result.kps.push_back(std::max(0.0f, pt.y - offsetY));
}
}
else {
result.box = bbox;
// Copy landmarks directly
result.kps.reserve(obj.landmarks.points.size() * 2);
for (const auto& pt : obj.landmarks.points) {
result.kps.push_back(pt.x);
result.kps.push_back(pt.y);
}
}
output.push_back(std::move(result));
}
return output;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOVFD::RunInference",
"Exception: " + std::string(e.what()),
__FILE__, __LINE__);
return {};
}
catch (...) {
_logger.LogFatal("ANSOVFD::RunInference", "Unknown fatal exception",
__FILE__, __LINE__);
return {};
}
}
ANSOVFD::~ANSOVFD() {
try {
Destroy();
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFD::Destroy", e.what(), __FILE__, __LINE__);
}
}
bool ANSOVFD::Destroy() {
try {
#ifdef USEOVFACEDETECTOR
_faceLandmark.reset(); // Releases previously allocated memory for face landmark
#endif
_face_detector.reset(); // Releases previously allocated memory for face detection
_isInitialized = false; // Reset initialization flag
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("ANSOVFD::Destroy", "Failed to release ANSOVFD Models", __FILE__, __LINE__);
}
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFD::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
#ifdef USEOVFACEDETECTOR
std::shared_ptr<ov::Model> ANSOVFD::read(const ov::Core& core, std::string pathToModel) {
std::shared_ptr<ov::Model> model = core.read_model(pathToModel);
model_input_height = model->input().get_shape()[3];
model_input_width = model->input().get_shape()[2];
ov::OutputVector outputs = model->outputs();
if (outputs.size() == 1) {
output = outputs.front().get_any_name();
const auto& outShape = outputs.front().get_shape();
if (outShape.size() != 4) {
this->_logger.LogFatal("ANSOVFD::read.", "Face Detection model output should have 4 dimentions", __FILE__, __LINE__);
return model;
}
objectSize = outputs.front().get_shape()[3];
if (objectSize != 7) {
this->_logger.LogFatal("ANSOVFD::read. ", "Face Detection model output layer should have 7 as a last dimension", __FILE__, __LINE__);
return model;
}
if (outShape[2] != ndetections) {
this->_logger.LogFatal("ANSOVFD::read. ", "Face Detection model output must contain 200 detections", __FILE__, __LINE__);
return model;
}
}
else {
for (const auto& out : outputs) {
const auto& outShape = out.get_shape();
if (outShape.size() == 2 && outShape.back() == 5) {
output = out.get_any_name();
if (outShape[0] != ndetections) {
this->_logger.LogFatal("ANSOVFD::read. ", "Face Detection model output must contain 200 detections", __FILE__, __LINE__);
return model;
}
objectSize = outShape.back();
}
else if (outShape.size() == 1 && out.get_element_type() == ov::element::i32) {
labels_output = out.get_any_name();
}
}
if (output.empty() || labels_output.empty()) {
this->_logger.LogFatal("ANSOVFD::read. ", "Face Detection model must contain either single DetectionOutput", __FILE__, __LINE__);
return model;
}
}
ov::preprocess::PrePostProcessor ppp(model);
ppp.input().tensor().
set_element_type(ov::element::u8).
set_layout("NHWC");
ppp.input().preprocess().convert_layout("NCHW");
ppp.output(output).tensor().set_element_type(ov::element::f32);
model = ppp.build();
ov::set_batch(model, 1);
inShape = model->input().get_shape();
return model;
}
bool ANSOVFD::InitFaceDetector() {
if (_isInitialized) {
_faceLandmark.reset(); // Releases previously allocated memory for face landmark
_isInitialized = false; // Reset initialization flag
}
// Check if the Yolo model is available
std::string yoloModel = CreateFilePath(_modelFolder, "ansylfd.xml");
if (std::filesystem::exists(yoloModel)) {
_modelFilePath = yoloModel;
_useYoloFaceDetector = true;
}
else {
_useYoloFaceDetector = false;
std::string xmlfile = CreateFilePath(_modelFolder, "ansovfd.xml");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
}
else {
this->_logger.LogError("ANSOVFD::Initialize. Face detector model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
std::string deviceName = GetOpenVINODevice();
std::string landmarkModel = CreateFilePath(_modelFolder, "landmarks.xml");
if (std::filesystem::exists(landmarkModel)) {
_landmarkModelFilePath = landmarkModel;
this->_logger.LogDebug("ANSOVFD::Initialize. Loading landmarks weight", _landmarkModelFilePath, __FILE__, __LINE__);
ov::Core lmcore;
CnnConfig landmarks_config(_landmarkModelFilePath, "Facial Landmarks Regression");
if (deviceName == "NPU") {
lmcore.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
lmcore.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
landmarks_config.m_deviceName = "AUTO:NPU,GPU";
}
else {
//lmcore.set_property(deviceName, ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
landmarks_config.m_deviceName = deviceName;
}
landmarks_config.m_max_batch_size = 1;
landmarks_config.m_core = lmcore;
this->_faceLandmark = std::make_unique<VectorCNN>(landmarks_config);
if (!_faceLandmark) {
this->_logger.LogFatal("ANSOVFD::Initialize", "Failed to initialize face recognizer model", __FILE__, __LINE__);
}
}
else {
this->_logger.LogError("ANSOVFD::LoadModel. Model landmarks.xml file is not exist", _landmarkModelFilePath, __FILE__, __LINE__);
}
if (_useYoloFaceDetector) InitialYoloModel(deviceName);
else InitialSSDModel(deviceName);
_isInitialized = true;
return _isInitialized;
}
void ANSOVFD::InitialSSDModel(const std::string& deviceName) {
try {
bb_enlarge_coefficient = 1.0;
bb_dx_coefficient = 1;
bb_dy_coefficient = 1;
ov::Core core;
if (deviceName == "NPU") {
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
ov::CompiledModel cml = core.compile_model(read(core, _modelFilePath), "AUTO:NPU,GPU");
request = cml.create_infer_request();
inTensor = request.get_input_tensor();
inTensor.set_shape(inShape);
}
else {
//ore.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
//core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
ov::CompiledModel cml = core.compile_model(read(core, _modelFilePath), deviceName);
request = cml.create_infer_request();
inTensor = request.get_input_tensor();
inTensor.set_shape(inShape);
}
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSOVFD::InitialSSDModel", e.what(), __FILE__, __LINE__);
}
}
void ANSOVFD::InitialYoloModel(const std::string& deviceName) {
try {
ov::Core core;
if (deviceName == "NPU") {
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "AUTO:NPU,GPU");
}
else {
// Configure and compile for individual device
//core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
//core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, deviceName);
}
request = compiled_model_.create_infer_request();
}
catch (const std::exception& e) {
this->_logger.LogFatal("OPENVINOOD::InitialModel", e.what(), __FILE__, __LINE__);
}
}
std::vector<Object> ANSOVFD::runOVInference(const cv::Mat& frame, const std::string& cameraId) {
std::vector<Object> detectionOutputs;
if (frame.empty()) {
this->_logger.LogError("ANSOVFD::runOVInference", "Input image is empty", __FILE__, __LINE__);
return detectionOutputs;
}
if ((frame.cols < 10) || (frame.rows < 10)) return detectionOutputs;
try {
width = static_cast<float>(frame.cols);
height = static_cast<float>(frame.rows);
resize2tensor(frame, inTensor);
request.infer();
float* detections = request.get_tensor(output).data<float>();
if (!labels_output.empty()) {
const int32_t* labels = request.get_tensor(labels_output).data<int32_t>();
for (size_t i = 0; i < ndetections; i++) {
Object r;
r.className = labels[i];
r.confidence = detections[i * objectSize + 4];
if (r.confidence <= _modelConfig.detectionScoreThreshold) {
continue;
}
r.box.x = static_cast<int>(detections[i * objectSize] / model_input_width * width);
r.box.y = static_cast<int>(detections[i * objectSize + 1] / model_input_height * height);
r.box.width = static_cast<int>(detections[i * objectSize + 2] / model_input_width * width - r.box.x);
r.box.height = static_cast<int>(detections[i * objectSize + 3] / model_input_height * height - r.box.y);
// Make square and enlarge face bounding box for more robust operation of face analytics networks
int bb_width = r.box.width;
int bb_height = r.box.height;
int bb_center_x = r.box.x + bb_width / 2;
int bb_center_y = r.box.y + bb_height / 2;
int max_of_sizes = std::max(bb_width, bb_height);
int bb_new_width = static_cast<int>(bb_enlarge_coefficient * max_of_sizes);
int bb_new_height = static_cast<int>(bb_enlarge_coefficient * max_of_sizes);
r.box.x = bb_center_x - static_cast<int>(std::floor(bb_dx_coefficient * bb_new_width / 2));
r.box.y = bb_center_y - static_cast<int>(std::floor(bb_dy_coefficient * bb_new_height / 2));
r.box.width = bb_new_width;
r.box.height = bb_new_height;
r.box.y = std::max(0, r.box.y);
r.box.x = std::max(0, r.box.x);
r.box.width = std::min(r.box.width, frame.cols - r.box.x);
r.box.height = std::min(r.box.height, frame.rows - r.box.y);
r.cameraId = cameraId;
if (r.confidence > _modelConfig.detectionScoreThreshold) {
detectionOutputs.push_back(r);
}
}
}
for (size_t i = 0; i < ndetections; i++) {
float image_id = detections[i * objectSize];
if (image_id < 0) {
break;
}
Object r;
r.className = static_cast<int>(detections[i * objectSize + 1]);
r.confidence = detections[i * objectSize + 2];
if (r.confidence <= _modelConfig.detectionScoreThreshold) {
continue;
}
r.box.x = static_cast<int>(detections[i * objectSize + 3] * width);
r.box.y = static_cast<int>(detections[i * objectSize + 4] * height);
r.box.width = static_cast<int>(detections[i * objectSize + 5] * width - r.box.x);
r.box.height = static_cast<int>(detections[i * objectSize + 6] * height - r.box.y);
// Make square and enlarge face bounding box for more robust operation of face analytics networks
int bb_width = r.box.width;
int bb_height = r.box.height;
int bb_center_x = r.box.x + bb_width / 2;
int bb_center_y = r.box.y + bb_height / 2;
int max_of_sizes = std::max(bb_width, bb_height);
int bb_new_width = static_cast<int>(bb_enlarge_coefficient * max_of_sizes);
int bb_new_height = static_cast<int>(bb_enlarge_coefficient * max_of_sizes);
r.box.x = bb_center_x - static_cast<int>(std::floor(bb_dx_coefficient * bb_new_width / 2));
r.box.y = bb_center_y - static_cast<int>(std::floor(bb_dy_coefficient * bb_new_height / 2));
r.box.width = bb_new_width;
r.box.height = bb_new_height;
r.box.y = std::max(0, r.box.y);
r.box.x = std::max(0, r.box.x);
r.box.width = std::min(r.box.width, frame.cols - r.box.x);
r.box.height = std::min(r.box.height, frame.rows - r.box.y);
r.cameraId = cameraId;
if (r.confidence > _modelConfig.detectionScoreThreshold) {
detectionOutputs.push_back(r);
}
}
return detectionOutputs;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFD::runOVInference", e.what(), __FILE__, __LINE__);
return detectionOutputs;
}
}
cv::Mat ANSOVFD::PreProcessing(const cv::Mat& source) {
try {
int col = source.cols;
int row = source.rows;
int _max = MAX(col, row);
cv::Mat result = cv::Mat::zeros(_max, _max, CV_8UC3);
if ((col > 0) && (row > 0))source.copyTo(result(cv::Rect(0, 0, col, row)));
return result;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVFD::PreProcessing", e.what(), __FILE__, __LINE__);
return source;
}
}
std::vector<Object> ANSOVFD::runYoloInference(const cv::Mat& input, const std::string& cameraId) {
std::vector<Object> outputs;
outputs.clear();
try {
if (input.empty()) {
this->_logger.LogError("ANSOVFD::RunInference", "Input image is empty", __FILE__, __LINE__);
return outputs;
}
if ((input.cols < 10) || (input.rows < 10)) return outputs;
// Step 0: Prepare input
cv::Mat img = input.clone();
cv::Mat letterbox_img = PreProcessing(img);
float scale = letterbox_img.size[0] / 640.0;
cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0, cv::Size(640, 640), cv::Scalar(), true);
// Step 1: Feed blob to the network
// Get input port for model with one input
auto input_port = compiled_model_.input();
// Create tensor from external memory
ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blob.ptr(0));
// Set input tensor for model with one input
request.set_input_tensor(input_tensor);
// Step 3. Start inference
request.infer();
// Step 4. Get output
auto output = request.get_output_tensor(0);
auto output_shape = output.get_shape();
int rows = output_shape[2]; //8400
int dimensions = output_shape[1]; //84: box[cx, cy, w, h]+80 classes scores
// Step 5. Post-processing
float* data = output.data<float>();
cv::Mat output_buffer(output_shape[1], output_shape[2], CV_32F, data);
transpose(output_buffer, output_buffer); //[8400,84]
std::vector<int> class_ids;
std::vector<float> class_scores;
std::vector<cv::Rect> boxes;
// Figure out the bbox, class_id and class_score
for (int i = 0; i < output_buffer.rows; i++) {
cv::Mat classes_scores = output_buffer.row(i).colRange(4, output_shape[1]);
cv::Point class_id;
double maxClassScore;
minMaxLoc(classes_scores, 0, &maxClassScore, 0, &class_id);
if (maxClassScore > _modelConfig.detectionScoreThreshold) {
class_scores.push_back(maxClassScore);
class_ids.push_back(class_id.x);
float cx = output_buffer.at<float>(i, 0);
float cy = output_buffer.at<float>(i, 1);
float w = output_buffer.at<float>(i, 2);
float h = output_buffer.at<float>(i, 3);
int left = int((cx - 0.5 * w) * scale);
int top = int((cy - 0.5 * h) * scale);
int width = int(w * scale);
int height = int(h * scale);
left = std::max(0, left);
top = std::max(0, top);
width = std::min(width, img.cols - left);
height = std::min(height, img.rows - top);
boxes.push_back(cv::Rect(left, top, width, height));
}
}
//NMS
std::vector<int> indices;
cv::dnn::NMSBoxes(boxes, class_scores, _modelConfig.modelConfThreshold, _modelConfig.modelMNSThreshold, indices);
for (int i = 0; i < indices.size(); i++)
{
Object result;
int id = indices[i];
if (class_scores[id] > _modelConfig.detectionScoreThreshold) {
int classId = class_ids[id];
result.classId = classId;
result.className = "Face";
result.confidence = class_scores[id];
result.box = boxes[id];
outputs.push_back(result);
}
}
img.release();
letterbox_img.release();
return outputs;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOOD::runYoloInference", e.what(), __FILE__, __LINE__);
return outputs;
}
}
#endif
}
+64
View File
@@ -0,0 +1,64 @@
#ifndef ANSOVFaceDetector_H
#define ANSOVFaceDetector_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
#include "cnn.hpp"
#include "ONNXEngine.h"
//#define USEOVFACEDETECTOR
//#define FACEDEBUG
namespace ANSCENTER {
class ANSENGINE_API ANSOVFD :public ANSFDBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input, bool useDynamicImage = true, bool validateFace=false, bool facelivenessCheck=true);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id, bool useDynamicImage = true, bool validateFace=false, bool facelivenessCheck=true);
bool Destroy();
~ANSOVFD();
private:
std::mutex _mtx;
std::string _modelFilePath;
std::unique_ptr<SCRFD> _face_detector = nullptr;
std::vector<Object> _movementObjects;
int _retainDetectedFaces{ 0 };
std::vector<Object> Inference(const cv::Mat& input, const std::string& camera_id, bool useDynamicImage = true, bool validateFace=false);
#ifdef USEOVFACEDETECTOR
std::string _landmarkModelFilePath;
std::unique_ptr<VectorCNN> _faceLandmark = nullptr;
ov::InferRequest request;
ov::Tensor inTensor;
ov::Shape inShape;
ov::CompiledModel compiled_model_;
const size_t ndetections = 200;
std::string output;
std::string labels_output;
bool _useYoloFaceDetector{true};
size_t objectSize;
float width;
float height;
size_t model_input_width;
size_t model_input_height;
float bb_enlarge_coefficient;
float bb_dx_coefficient;
float bb_dy_coefficient;
std::shared_ptr<ov::Model> read(const ov::Core& core, std::string pathToModel);
std::vector<Object> runOVInference(const cv::Mat& frame, const std::string& cameraId);
void InitialSSDModel(const std::string& deviceName);
void InitialYoloModel(const std::string& deviceName);
cv::Mat PreProcessing(const cv::Mat& source);
std::vector<Object> runYoloInference(const cv::Mat& input, const std::string& cameraId);
cv::Mat resized_frame_;
cv::Point2f factor_;
cv::Size2f model_input_shape_;
cv::Size model_output_shape_;
bool InitFaceDetector();
#endif
};
}
#endif
+792
View File
@@ -0,0 +1,792 @@
#include "ANSOVSEG.h"
namespace ANSCENTER {
bool ANSOVSEG::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("ANSOVSEG::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("ANSOVSEG::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("ANSOVSEG::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool ANSOVSEG::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("ANSOVSEG::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVSEG::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSOVSEG::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSOVSEG::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVSEG::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".xml";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::SEGMENTATION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, modelFullName);//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, className);
this->_logger.LogDebug("ANSOVSEG::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVSEG::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSOVSEG::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSOVSEG::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("OPENVINOCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVSEG::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
std::string openVINOVersion = ov::get_openvino_version().buildNumber;
this->_logger.LogDebug("ANSOVSEG::Initialize. OpenVINO version", openVINOVersion, __FILE__, __LINE__);
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::SEGMENTATION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("ANSOVSEG::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVSEG::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSOVSEG::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSOVSEG::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel();
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVSEG::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSOVSEG::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> ANSOVSEG::RunInference(const cv::Mat& input,const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Validation
if (!_licenseValid) {
_logger.LogError("ANSOVSEG::RunInference", "Invalid License",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("ANSOVSEG::RunInference",
"Model is not initialized", __FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
return {};
}
// Preprocessing
cv::Mat letterbox_img = PreProcessing(input);
if (letterbox_img.empty()) {
_logger.LogError("ANSOVSEG::RunInference", "PreProcessing failed",
__FILE__, __LINE__);
return {};
}
constexpr int imageSize = 640;
const float scale = static_cast<float>(letterbox_img.rows) / imageSize;
cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0,
cv::Size(imageSize, imageSize),
cv::Scalar(), true);
// Inference
auto input_port = compiled_model_.input();
ov::Tensor input_tensor(input_port.get_element_type(),
input_port.get_shape(), blob.ptr(0));
inference_request_.set_input_tensor(input_tensor);
inference_request_.infer();
// Get outputs
auto output0 = inference_request_.get_output_tensor(0);
auto output1 = inference_request_.get_output_tensor(1);
auto output0_shape = output0.get_shape();
const int rows = static_cast<int>(output0_shape[2]); // 8400
const int dimensions = static_cast<int>(output0_shape[1]); // 116
float* data = output0.data<float>();
// Proto masks [32, 25600]
cv::Mat proto(32, 25600, CV_32F, output1.data<float>());
// Parse detections (NO TRANSPOSE!)
std::vector<int> class_ids;
std::vector<float> class_scores;
std::vector<cv::Rect> boxes;
std::vector<cv::Mat> mask_confs;
// Pre-allocate
class_ids.reserve(rows / 10);
class_scores.reserve(rows / 10);
boxes.reserve(rows / 10);
mask_confs.reserve(rows / 10);
const cv::Rect imageBounds(0, 0, input.cols, input.rows);
// Process detections without transpose
for (int i = 0; i < rows; i++) {
// Find max class score (columns 4-83, total 80 classes)
float max_score = -1.0f;
int max_class_id = 0;
for (int j = 4; j < 84; j++) {
const float score = data[j * rows + i];
if (score > max_score) {
max_score = score;
max_class_id = j - 4;
}
}
if (max_score > _modelConfig.detectionScoreThreshold) {
// Extract box coordinates
const float cx = data[0 * rows + i];
const float cy = data[1 * rows + i];
const float w = data[2 * rows + i];
const float h = data[3 * rows + i];
// Convert to pixel coordinates
cv::Rect box(
static_cast<int>((cx - 0.5f * w) * scale),
static_cast<int>((cy - 0.5f * h) * scale),
static_cast<int>(w * scale),
static_cast<int>(h * scale)
);
box &= imageBounds; // Clamp to bounds
if (box.area() > 0) {
class_ids.push_back(max_class_id);
class_scores.push_back(max_score);
boxes.push_back(box);
// Extract mask coefficients (columns 84-115, 32 values)
cv::Mat mask_conf(1, 32, CV_32F);
for (int k = 0; k < 32; k++) {
mask_conf.at<float>(0, k) = data[(84 + k) * rows + i];
}
mask_confs.push_back(mask_conf);
}
}
}
if (boxes.empty()) {
return {};
}
// Apply NMS
std::vector<int> indices;
cv::dnn::NMSBoxes(boxes, class_scores,
_modelConfig.detectionScoreThreshold,
_modelConfig.modelMNSThreshold, indices);
// Process final detections
std::vector<Object> outputs;
outputs.reserve(indices.size());
const int classNameSize = static_cast<int>(_classes.size());
for (int idx : indices) {
// Generate mask for this detection
cv::Mat m = mask_confs[idx] * proto; // [1, 25600]
// Apply sigmoid vectorized
cv::exp(-m, m);
m = 1.0 / (1.0 + m);
// Reshape to 160x160
cv::Mat m1 = m.reshape(1, 160);
// Get ROI coordinates
const cv::Rect& box = boxes[idx];
int x1 = std::max(0, box.x);
int y1 = std::max(0, box.y);
int x2 = std::min(input.cols, box.br().x);
int y2 = std::min(input.rows, box.br().y);
if (x2 <= x1 || y2 <= y1) continue; // Invalid box
// Calculate mask ROI (in 160x160 space, 0.25 scale)
int mx1 = static_cast<int>(x1 / scale * 0.25);
int my1 = static_cast<int>(y1 / scale * 0.25);
int mx2 = static_cast<int>(x2 / scale * 0.25);
int my2 = static_cast<int>(y2 / scale * 0.25);
// Clamp mask coordinates
mx1 = std::clamp(mx1, 0, 160);
my1 = std::clamp(my1, 0, 160);
mx2 = std::clamp(mx2, 0, 160);
my2 = std::clamp(my2, 0, 160);
if (mx2 <= mx1 || my2 <= my1) continue; // Invalid mask ROI
// Extract and resize mask
cv::Mat mask_roi = m1(cv::Range(my1, my2), cv::Range(mx1, mx2));
cv::Mat rm;
cv::resize(mask_roi, rm, cv::Size(x2 - x1, y2 - y1));
// Threshold mask (vectorized)
cv::threshold(rm, rm, 0.5, 1.0, cv::THRESH_BINARY);
// Convert to uint8
cv::Mat det_mask;
rm.convertTo(det_mask, CV_8UC1, 255.0);
// Create full-size mask
cv::Mat bbox_mask = cv::Mat::zeros(input.size(), CV_8UC1);
det_mask.copyTo(bbox_mask(cv::Rect(x1, y1, x2 - x1, y2 - y1)));
// Extract contours
std::vector<std::vector<cv::Point>> contours;
cv::findContours(bbox_mask, contours, cv::RETR_EXTERNAL,
cv::CHAIN_APPROX_SIMPLE);
if (contours.empty()) continue; // No contour found
// Find largest contour
size_t max_contour_idx = 0;
double max_area = 0;
for (size_t i = 0; i < contours.size(); i++) {
double area = cv::contourArea(contours[i]);
if (area > max_area) {
max_area = area;
max_contour_idx = i;
}
}
// Normalize polygon
std::vector<cv::Point2f> normalizedPolygon;
normalizedPolygon.reserve(contours[max_contour_idx].size());
for (const auto& pt : contours[max_contour_idx]) {
normalizedPolygon.emplace_back(
static_cast<float>(pt.x),
static_cast<float>(pt.y)
);
}
// Build result object
Object result;
result.classId = class_ids[idx];
result.confidence = class_scores[idx];
result.box = box;
result.mask = det_mask.clone(); // Only clone final mask
result.polygon = std::move(normalizedPolygon);
result.cameraId = camera_id;
// Set class name
if (!_classes.empty()) {
result.className = (result.classId < classNameSize)
? _classes[result.classId]
: _classes.back();
}
else {
result.className = "Unknown";
}
outputs.push_back(std::move(result));
}
if (_trackerEnabled) {
outputs = ApplyTracking(outputs, camera_id);
if (_stabilizationEnabled) outputs = StabilizeDetections(outputs, camera_id);
}
return outputs;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOVSEG::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
ANSOVSEG::~ANSOVSEG() {
try {
// We will do it all togther when exit ANSVIS
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("ANSOVSEG::~ANSOVSEG()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
}
}
}
catch (std::exception& e) {
std::cout << "ANSOVSEG::~ANSOVSEG()" << e.what() << std::endl;
}
}
bool ANSOVSEG::Destroy() {
try {
// We will do it all togther when exit ANSVIS
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("ANSOVSEG::Destroy()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
}
}
return true;
}
catch (std::exception& e) {
std::cout << "ANSOVSEG::Destroy()" << e.what() << std::endl;
return false;
}
}
//private
void ANSOVSEG::InitialModel() {
try {
// Step 1: Initialize OpenVINO Runtime Core
ov::Core core;
// Step 2: Get Available Devices and Log
std::vector<std::string> available_devices = core.get_available_devices();
// Define device priority: NPU > GPU > CPU
std::vector<std::string> priority_devices = { "NPU", "GPU" };
bool device_found = false;
// Iterate over prioritized devices
for (const auto& device : priority_devices) {
if (std::find(available_devices.begin(), available_devices.end(), device) != available_devices.end()) {
if (device == "NPU") {
core.set_property("NPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
core.set_property("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "AUTO:NPU,GPU");
}
else {
// Configure and compile for individual device
core.set_property(device, ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, device);
}
device_found = true;
break;
}
}
// Fallback: Default to CPU if no devices found
if (!device_found) {
core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
compiled_model_ = core.compile_model(_modelFilePath, "CPU");
}
// Step 3: Create Inference Request
inference_request_ = compiled_model_.create_infer_request();
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVSEG::InitialModel", e.what(), __FILE__, __LINE__);
}
}
cv::Mat ANSOVSEG::PreProcessing(const cv::Mat& source) {
try {
if (source.empty()) {
this->_logger.LogFatal("ANSOVSEG::PreProcessing", "Empty image provided", __FILE__, __LINE__);
return cv::Mat();
}
// Convert grayscale to 3-channel BGR if needed
cv::Mat processedImage;
if (source.channels() == 1) {
cv::cvtColor(source, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = source;
}
int col = processedImage.cols;
int row = processedImage.rows;
int maxSize = std::max(col, row);
// Create a square padded image with a black background
cv::Mat result = cv::Mat::zeros(maxSize, maxSize, CV_8UC3);
// Copy the original image to the top-left corner of the square matrix
processedImage.copyTo(result(cv::Rect(0, 0, col, row)));
return result;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSOVSEG::PreProcessing", e.what(), __FILE__, __LINE__);
return cv::Mat();
}
}
cv::Rect ANSOVSEG::GetBoundingBox(const cv::Rect& src) {
cv::Rect box = src;
box.x = (box.x - 0.5 * box.width) * factor_.x;
box.y = (box.y - 0.5 * box.height) * factor_.y;
box.width *= factor_.x;
box.height *= factor_.y;
return box;
}
std::vector<Object>ANSOVSEG::PostProcessing(const std::string& camera_id) {
try {
std::vector<int> class_list;
std::vector<float> confidence_list;
std::vector<cv::Rect> box_list;
float* detections = inference_request_.get_output_tensor().data<float>();
const cv::Mat detection_outputs(model_output_shape_, CV_32F, (float*)detections);
for (int i = 0; i < detection_outputs.cols; ++i) {
const cv::Mat classes_scores = detection_outputs.col(i).rowRange(4, detection_outputs.rows);
cv::Point class_id;
double score;
cv::minMaxLoc(classes_scores, nullptr, &score, nullptr, &class_id);
if (score >= _modelConfig.detectionScoreThreshold) {
class_list.push_back(class_id.y);
confidence_list.push_back(score);
const float x = detection_outputs.at<float>(0, i);
const float y = detection_outputs.at<float>(1, i);
const float w = detection_outputs.at<float>(2, i);
const float h = detection_outputs.at<float>(3, i);
cv::Rect box;
box.x = static_cast<int>(x);
box.y = static_cast<int>(y);
box.width = static_cast<int>(w);
box.height = static_cast<int>(h);
box_list.push_back(box);
}
}
std::vector<int> NMS_result;
cv::dnn::NMSBoxes(box_list, confidence_list, _modelConfig.modelConfThreshold, _modelConfig.modelMNSThreshold, NMS_result);
std::vector<Object> output;
for (int i = 0; i < NMS_result.size(); i++)
{
Object result;
int id = NMS_result[i];
result.classId = class_list[id];
result.confidence = confidence_list[id];
result.box = GetBoundingBox(box_list[id]);
result.cameraId = camera_id;
output.push_back(result);
}
//EnqueueDetection(output,camera_id);
return output;
}
catch (const std::exception& e) {
std::vector<Object> result;
result.clear();
this->_logger.LogError("ANSOVSEG::PostprocessImage", e.what(), __FILE__, __LINE__);
return result;
}
}
float ANSOVSEG::sigmoid_function(float a) {
float b = 1. / (1. + exp(-a));
return b;
}
}
// Only work for Yolov8 for now, extract polygons does not work as expected
//std::vector<Object> ANSOVSEG::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> outputs;
// outputs.clear();
// if (!_licenseValid) {
// this->_logger.LogError("ANSOVSEG::RunInference", "Invalid License", __FILE__, __LINE__);
// return outputs;
// }
// if (!_isInitialized) {
// this->_logger.LogError("ANSOVSEG::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return outputs;
// }
// try {
// // Step 0: Prepare input
// if (input.empty()) return outputs;
// if ((input.cols < 10) || (input.rows < 10)) return outputs;
// cv::Mat letterbox_img = PreProcessing(input);
// if (letterbox_img.empty()) {
// _logger.LogError("OPENVINOOD::RunInference", "PreProcessing failed", __FILE__, __LINE__);
// return outputs;
// }
// int imageSize = 640;
// int maxImageSize = std::max(letterbox_img.cols, letterbox_img.rows);
// //if (maxImageSize < imageSize)imageSize = maxImageSize;
// float scale = static_cast<float>(letterbox_img.rows) / imageSize;
// cv::Mat blob = cv::dnn::blobFromImage(letterbox_img, 1.0 / 255.0,cv::Size(imageSize, imageSize),cv::Scalar(), true);
// // Step 1: Feed blob to the network
// auto input_port = compiled_model_.input();
// ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blob.ptr(0));
// inference_request_.set_input_tensor(input_tensor);
// inference_request_.infer();
// // Step 2: Get output
// auto output0 = inference_request_.get_output_tensor(0); //output0
// auto output1 = inference_request_.get_output_tensor(1); //otuput1
// auto output0_shape = output0.get_shape();
// auto output1_shape = output1.get_shape();
//
// // Step 3: Postprocess the inference result
// cv::Mat output_buffer(output0_shape[1], output0_shape[2], CV_32F, output0.data<float>());
// cv::Mat proto(32, 25600, CV_32F, output1.data<float>()); //[32,25600]
// transpose(output_buffer, output_buffer); //[8400,116]
//
// std::vector<int> class_ids;
// std::vector<float> class_scores;
// std::vector<cv::Rect> boxes;
// std::vector<cv::Mat> mask_confs;
// // Figure out the bbox, class_id and class_score
// for (int i = 0; i < output_buffer.rows; i++) {
// cv::Mat classes_scores = output_buffer.row(i).colRange(4, 84);
// cv::Point class_id;
// double maxClassScore;
// minMaxLoc(classes_scores, 0, &maxClassScore, 0, &class_id);
// if (maxClassScore > _modelConfig.detectionScoreThreshold) {
// class_scores.push_back(maxClassScore);
// class_ids.push_back(class_id.x);
// float cx = output_buffer.at<float>(i, 0);
// float cy = output_buffer.at<float>(i, 1);
// float w = output_buffer.at<float>(i, 2);
// float h = output_buffer.at<float>(i, 3);
// int left = int((cx - 0.5 * w) * scale);
// int top = int((cy - 0.5 * h) * scale);
// int width = int(w * scale);
// int height = int(h * scale);
// cv::Mat mask_conf = output_buffer.row(i).colRange(84, 116);
// mask_confs.push_back(mask_conf);
// boxes.push_back(cv::Rect(left, top, width, height));
// }
// }
// //NMS
// std::vector<int> indices;
// cv::dnn::NMSBoxes(boxes, class_scores, _modelConfig.detectionScoreThreshold, _modelConfig.modelMNSThreshold, indices);
// int classNameSize = static_cast<int>(_classes.size());
// for (size_t i = 0; i < indices.size(); i++) {
// // Visualize the objects
// if (class_scores[i] >= _modelConfig.detectionScoreThreshold) {
// int index = indices[i];
// // Visualize the Masks
// cv::Mat m = mask_confs[i] * proto;
// for (int col = 0; col < m.cols; col++) {
// m.at<float>(0, col) = sigmoid_function(m.at<float>(0, col));
// }
// cv::Mat m1 = m.reshape(1, 160); // 1x25600 -> 160x160
// int x1 = std::max(0, boxes[index].x);
// int y1 = std::max(0, boxes[index].y);
// int x2 = std::min(input.cols, boxes[index].br().x);
// int y2 = std::min(input.rows, boxes[index].br().y);
// int mx1 = int(x1 / scale * 0.25);
// int my1 = int(y1 / scale * 0.25);
// int mx2 = int(x2 / scale * 0.25);
// int my2 = int(y2 / scale * 0.25);
// cv::Mat mask_roi = m1(cv::Range(my1, my2), cv::Range(mx1, mx2));
// cv::Mat rm, det_mask;
// cv::resize(mask_roi, rm, cv::Size(x2 - x1, y2 - y1));
// for (int r = 0; r < rm.rows; r++) {
// for (int c = 0; c < rm.cols; c++) {
// float pv = rm.at<float>(r, c);
// rm.at<float>(r, c) = (pv > 0.5) ? 1.0 : 0.0;
// }
// }
// rm = rm * 255; // Scale mask to 255 for visualization
// rm.convertTo(det_mask, CV_8UC1);
// // Extract contours from the bounding box region
// cv::Mat bbox_mask = cv::Mat::zeros(input.size(), CV_8UC1);
// det_mask.copyTo(bbox_mask(cv::Rect(x1, y1, x2 - x1, y2 - y1)));
// std::vector<std::vector<cv::Point>> contours;
// cv::findContours(bbox_mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
// // Normalize the polygon
// std::vector<cv::Point2f> normalizedPolygon;
// for (const auto& pt : contours[0]) {
// normalizedPolygon.emplace_back(static_cast<float>(pt.x) / 1.0, static_cast<float>(pt.y) / 1.0);
// }
// // add to outputs
// Object result;
// int class_id = class_ids[i];
// result.classId = class_id;
// if (!_classes.empty()) {
// if (result.classId < classNameSize) {
// result.className = _classes[result.classId];
// }
// else {
// result.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
// }
// }
// else {
// result.className = "Unknown"; // Fallback if _classes is empty
// }
// result.confidence = class_scores[i];
// result.box = boxes[i];
// result.mask = det_mask(cv::Rect(x1, y1, x2 - x1, y2 - y1)).clone();
// result.polygon = normalizedPolygon;
// result.cameraId = camera_id;
// outputs.push_back(result);
//
// }
// }
// return outputs;
// }
// catch (std::exception& e) {
// this->_logger.LogFatal("ANSOVSEG::RunInference", e.what(), __FILE__, __LINE__);
// return outputs;
// }
//}
+48
View File
@@ -0,0 +1,48 @@
#ifndef ANSOVSEG_H
#define ANSOVSEG_H
#pragma once
#include "ANSEngineCommon.h"
#include <string>
#include <opencv2/opencv.hpp>
#include <openvino/openvino.hpp>
#include <array>
namespace ANSCENTER
{
class ANSENGINE_API ANSOVSEG : public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSOVSEG();
private:
std::string _modelFilePath;
private:
void InitialModel();
cv::Mat PreProcessing(const cv::Mat& source);
cv::Rect GetBoundingBox(const cv::Rect& src);
std::vector<Object>PostProcessing(const std::string& camera_id);
float sigmoid_function(float a);
cv::Mat resized_frame_;
cv::Point2f factor_;
cv::Size2f model_input_shape_;
cv::Size model_output_shape_;
ov::Tensor input_tensor_;
ov::InferRequest inference_request_;
ov::CompiledModel compiled_model_;
// For visualization
std::vector<cv::Scalar> colors = { cv::Scalar(255, 0, 0), cv::Scalar(255, 0, 255), cv::Scalar(170, 0, 255), cv::Scalar(255, 0, 85),
cv::Scalar(255, 0, 170), cv::Scalar(85, 255, 0), cv::Scalar(255, 170, 0), cv::Scalar(0, 255, 0),
cv::Scalar(255, 255, 0), cv::Scalar(0, 255, 85), cv::Scalar(170, 255, 0), cv::Scalar(0, 85, 255),
cv::Scalar(0, 255, 170), cv::Scalar(0, 0, 255), cv::Scalar(0, 255, 255), cv::Scalar(85, 0, 255) };
};
}
#endif
+541
View File
@@ -0,0 +1,541 @@
#include "ANSPOSE.h"
#include <pipelines/metadata.h>
#include <models/input_data.h>
#include <models/results.h>
namespace ANSCENTER {
bool ANSPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("ANSPOSE::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("ANSPOSE::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("ANSPOSE::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool ANSPOSE::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_device = GetOpenVINODevice(_core);
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlfile = CreateFilePath(_modelFolder, "ansposesetimation.xml");
std::string binaryfile = CreateFilePath(_modelFolder, "ansposesetimation.bin");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSPOSE::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSPOSE::Initialize binary weight", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model binary file does not exist", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model file does exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSPOSE::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "ansposesetimation";
}
std::string xmlFileName = _modelName + ".xml";
std::string binFileName = _modelName + ".bin";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::KEYPOINT;
_modelConfig.modelType = ModelType::POSE;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
_device = GetOpenVINODevice(_core);
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlfile = CreateFilePath(_modelFolder, xmlFileName);
std::string binaryfile = CreateFilePath(_modelFolder, binFileName);
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSPOSE::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSPOSE::Initialize binary weight", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model binary file does not exist", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model file does exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
labelMap = "Person";
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSPOSE::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
std::string openVINOVersion = ov::get_openvino_version().buildNumber;
this->_logger.LogDebug("ANSPOSE::Initialize. OpenVINO version", openVINOVersion, __FILE__, __LINE__);
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for ABNORMAL only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::KEYPOINT;
_modelConfig.modelType = ModelType::POSE;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
_device = GetOpenVINODevice(_core);
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlfile = CreateFilePath(_modelFolder, "ansposesetimation.xml");
std::string binaryfile = CreateFilePath(_modelFolder, "ansposesetimation.bin");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSPOSE::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSPOSE::Initialize binary weight", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model binary file does not exist", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model file does exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
labelMap = "Person";
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSANOMALIB::Initialize", e.what(), __FILE__, __LINE__);
labelMap = "Normal,Defect";
return false;
}
}
std::vector<Object> ANSPOSE::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> ANSPOSE::RunInference(const cv::Mat& input,
const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Validation
if (!_licenseValid) {
_logger.LogError("ANSPOSE::RunInference", "Invalid License",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("ANSPOSE::RunInference", "Model is not initialized",
__FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
return {};
}
// Convert to BGR if needed
cv::Mat processedImage;
if (input.channels() == 1) {
cv::cvtColor(input, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = input; // Shallow copy
}
const int width = processedImage.cols;
const int height = processedImage.rows;
// Initialize pipeline once
if (!_poseEstimationPipeline) {
double aspectRatio = width / static_cast<double>(height);
auto model = std::make_unique<HPEOpenPose>(
_modelFilePath, aspectRatio, 0, 0.5f, ""
);
try {
_poseEstimationPipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig(_device, 1, "", 4), _core);
}
catch (const std::exception& e) {
// Fall back to CPU
_logger.LogWarn("ANSPOSE::RunInference",
"GPU init failed, using CPU: " + std::string(e.what()),
__FILE__, __LINE__);
model = std::make_unique<HPEOpenPose>(
_modelFilePath, aspectRatio, 0, 0.5f, ""
);
_poseEstimationPipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig("CPU", 0, "", 0), _core);
}
}
if (!_poseEstimationPipeline) {
_logger.LogError("ANSPOSE::RunInference", "Pipeline initialization failed",
__FILE__, __LINE__);
return {};
}
// Run inference
auto startTime = std::chrono::steady_clock::now();
int64_t frameNum = _poseEstimationPipeline->submitData(
ImageInputData(processedImage),
std::make_shared<ImageMetaData>(processedImage, startTime)
);
_poseEstimationPipeline->waitForTotalCompletion();
std::unique_ptr<ResultBase> result = _poseEstimationPipeline->getResult();
// Process results
std::vector<Object> output;
auto& poses = result->asRef<HumanPoseResult>().poses;
output.reserve(poses.size());
const cv::Rect imageBounds(0, 0, width, height);
for (const auto& pose : poses) {
if (pose.keypoints.empty()) {
continue;
}
// Calculate bounding box from keypoints
int x_min = width, y_min = height;
int x_max = 0, y_max = 0;
for (const auto& kp : pose.keypoints) {
if (kp.x > 0) {
x_min = std::min(x_min, static_cast<int>(kp.x));
x_max = std::max(x_max, static_cast<int>(kp.x));
}
if (kp.y > 0) {
y_min = std::min(y_min, static_cast<int>(kp.y));
y_max = std::max(y_max, static_cast<int>(kp.y));
}
}
// Validate bounding box
if (x_max <= x_min || y_max <= y_min) {
continue;
}
Object obj;
obj.classId = 0;
obj.className = "Person";
obj.confidence = 1.0f;
obj.cameraId = camera_id;
// Set bounding box
obj.box = cv::Rect(x_min, y_min, x_max - x_min, y_max - y_min);
obj.box &= imageBounds; // Clamp to image bounds
// Extract mask ROI (CRITICAL: Clone to avoid dangling reference!)
cv::Rect maskRect = obj.box & imageBounds;
if (maskRect.area() > 0) {
obj.mask = processedImage(maskRect).clone(); // Must clone!
}
// Store keypoints efficiently
obj.kps.reserve(pose.keypoints.size() * 2);
obj.polygon.reserve(pose.keypoints.size());
for (const auto& kp : pose.keypoints) {
obj.kps.push_back(kp.x);
obj.kps.push_back(kp.y);
obj.polygon.emplace_back(kp.x, kp.y);
}
// Build extraInfo string efficiently (if really needed)
if (!pose.keypoints.empty()) {
std::string keypointX, keypointY;
keypointX.reserve(pose.keypoints.size() * 10); // Estimate
keypointY.reserve(pose.keypoints.size() * 10);
for (size_t i = 0; i < pose.keypoints.size(); i++) {
keypointX += std::to_string(pose.keypoints[i].x);
keypointY += std::to_string(pose.keypoints[i].y);
if (i < pose.keypoints.size() - 1) {
keypointX += ";";
keypointY += ";";
}
}
obj.extraInfo = keypointX + "|" + keypointY;
}
output.push_back(std::move(obj));
}
if (_trackerEnabled) {
output = ApplyTracking(output, camera_id);
if (_stabilizationEnabled) output = StabilizeDetections(output, camera_id);
}
return output;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSPOSE::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
ANSPOSE::~ANSPOSE() {
if (_poseEstimationPipeline) {
delete _poseEstimationPipeline;
_poseEstimationPipeline = nullptr;
}
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("ANSPOSE::~ANSPOSE", "Failed to release ANSPOSE Models", __FILE__, __LINE__);
}
}
}
bool ANSPOSE::Destroy() {
if (_poseEstimationPipeline) {
delete _poseEstimationPipeline;
_poseEstimationPipeline = nullptr;
}
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogError("ANSPOSE::Destroy", "Failed to release ANSPOSE Models", __FILE__, __LINE__);
}
}
return true;
}
}
// std::vector<Object> ANSPOSE::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> output;
// output.clear();
// if (!_licenseValid) {
// this->_logger.LogError("ANSPOSE::RunInference", "Invalid License", __FILE__, __LINE__);
// return output;
// }
// if (!_isInitialized) {
// this->_logger.LogError("ANSPOSE::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return output;
// }
// try {
// if (input.empty()) return output;
// if ((input.cols < 10) || (input.rows < 10)) return output;
// // Convert grayscale to 3-channel BGR if needed
// cv::Mat processedImage;
// if (input.channels() == 1) {
// cv::cvtColor(input, processedImage, cv::COLOR_GRAY2BGR);
// }
// else {
// processedImage = input;
// }
// cv::Mat im = processedImage;
// int width = im.cols;
// int height = im.rows;
// if (!_poseEstimationPipeline) {
// // Load Model from Here
// std::unique_ptr<ModelBase> model;
// double aspectRatio = width / static_cast<double>(height);
// model.reset(new HPEOpenPose(_modelFilePath, aspectRatio, 0, static_cast<float>(0.5), ""));
// //ov::Core core;
// try {
// _poseEstimationPipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig(_device, 1, "", 4), _core);
// }
// catch (const std::exception& e) {
// // Fall back to CPU
// _poseEstimationPipeline = new AsyncPipeline(std::move(model), ConfigFactory::getUserConfig("CPU", 0, "", 0), _core);
// }
// }
// if (_poseEstimationPipeline) {
// auto startTime = std::chrono::steady_clock::now();
// int64_t frameNum = _poseEstimationPipeline->submitData(ImageInputData(im), std::make_shared<ImageMetaData>(im, startTime));
// _poseEstimationPipeline->waitForTotalCompletion();
// std::unique_ptr<ResultBase> result = _poseEstimationPipeline->getResult();
// /* static const std::pair<int, int> keypointsOP[] = {
// {1, 2},
// {1, 5},
// {2, 3},
// {3, 4},
// {5, 6},
// {6, 7},
// {1, 8},
// {8, 9},
// {9, 10},
// {1, 11},
// {11, 12},
// {12, 13},
// {1, 0},
// {0, 14},
// {14, 16},
// {0, 15},
// {15, 17}
// };*/
// const cv::Point2f absentKeypoint(-1.0f, -1.0f);
// for (auto& obj : result->asRef<HumanPoseResult>().poses) {
// Object result;
// float confidence = 1.0;
// if (confidence > 0)
// {
// // Intepret keypoints
// int x_min = 10000;
// int y_min = 10000;
// int x_max = 0;
// int y_max = 0;
// std::stringstream keypointXss;
// std::stringstream keypointYss;
// std::vector<cv::Point2f> kps;
// std::vector<float> keypoints;
// if (obj.keypoints.size() > 0) {
// for (size_t keypointIdx = 0; keypointIdx < obj.keypoints.size(); keypointIdx++) {
// if (obj.keypoints[keypointIdx].x > 0) {
// if (x_min > obj.keypoints[keypointIdx].x) x_min = obj.keypoints[keypointIdx].x;
// if (x_max < obj.keypoints[keypointIdx].x) x_max = obj.keypoints[keypointIdx].x;
// }
// if (obj.keypoints[keypointIdx].y > 0) {
// if (y_min > obj.keypoints[keypointIdx].y) y_min = obj.keypoints[keypointIdx].y;
// if (y_max < obj.keypoints[keypointIdx].y) y_max = obj.keypoints[keypointIdx].y;
// }
// keypointXss << obj.keypoints[keypointIdx].x; // Add semicolon after each x value
// keypointYss << obj.keypoints[keypointIdx].y; // Add semicolon after each y value
// // Add semicolon after each value except the last one
// if (keypointIdx < obj.keypoints.size() - 1)
// {
// keypointXss << ";";
// keypointYss << ";";
// }
// cv::Point2f kp;
// kp.x = obj.keypoints[keypointIdx].x;
// kp.y = obj.keypoints[keypointIdx].y;
// kps.push_back(kp);
// keypoints.push_back(kp.x);
// keypoints.push_back(kp.y);
// }
// // Retrieve strings from string streams
// std::string keypointXString = keypointXss.str();
// std::string keypointYString = keypointYss.str();
// std::string keypointString = keypointXString + "|" + keypointYString;
// result.classId = 0; // human keypoint
// result.className = "Person";
// result.confidence = confidence;
// result.box.x = x_min;
// result.box.y = y_min;
// result.box.width = x_max - x_min;
// result.box.height = y_max - y_min;
// result.box.x = std::max(0, result.box.x);
// result.box.y = std::max(0, result.box.y);
// result.box.width = std::min(width - result.box.x, result.box.width);
// result.box.height = std::min(height - result.box.y, result.box.height);
// cv::Rect maskImage(cv::Point(x_min, y_min), cv::Point(x_max, y_max));
// result.mask = im(maskImage);
// result.polygon = kps;
// result.kps = keypoints;
// result.extraInfo = keypointString;//Convert keypoint to st;
// result.cameraId = camera_id;
// output.push_back(result);
// }
// }
// }
// im.release();
// }
////EnqueueDetection(output,camera_id);
// return output;
// }
// catch (std::exception& e) {
// this->_logger.LogFatal("ANSPOSE::RunInference", e.what(), __FILE__, __LINE__);
// return output;
// }
// }
+33
View File
@@ -0,0 +1,33 @@
#ifndef ANSPOSE_H
#define ANSPOSE_H
#pragma once
#include "ANSEngineCommon.h"
#include <opencv2/opencv.hpp>
#include <opencv2/dnn.hpp>
#include <openvino/openvino.hpp>
#include <models/hpe_model_openpose.h>
#include <pipelines/async_pipeline.h>
namespace ANSCENTER
{
class ANSENGINE_API ANSPOSE :public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSPOSE();
private:
AsyncPipeline* _poseEstimationPipeline = nullptr;
std::string _modelFilePath;
std::string _device;
ov::Core _core;
};
}
#endif
File diff suppressed because it is too large Load Diff
+166
View File
@@ -0,0 +1,166 @@
#ifndef ANSRTYOLO_H
#define ANSRTYOLO_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
#include "engine/EnginePoolManager.h"
namespace ANSCENTER {
// ====================================================================
// ANSRTYOLO — TensorRT-based Ultralytics YOLO inference
//
// Compatible with ALL Ultralytics YOLO tasks and versions
// (v8, v9, v10, v11, v26+) by auto-detecting the task type
// from output tensor shapes at inference time.
//
// Supported tasks:
// - Detection : legacy [B, nc+4, N] or end2end [B, 300, 6]
// - OBB : legacy [B, nc+5, N] or end2end [B, 300, 7]
// - Segmentation: legacy [B, nc+36, N]+protos or end2end [B, 300, 38]+protos
// - Pose : legacy [B, nc+4+nk*3, N] or end2end [B, 300, 6+nk*3]
// - Classification: [B, nc]
//
// Uses Engine<float> with GPU preprocessing (cv::cuda::GpuMat),
// supports multi-GPU pool via SetMaxSlotsPerGpu.
// ====================================================================
class ANSENGINE_API ANSRTYOLO : public ANSODBase {
public:
bool Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath,
const std::string& modelZipPassword,
std::string& labelMap) override;
bool LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword) override;
bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap) override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override;
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs, const std::string& camera_id) override;
bool Destroy();
~ANSRTYOLO();
private:
std::string _modelFilePath;
bool _modelLoadValid{ false };
bool _fp16{ false };
bool _isFixedBatch{ false };
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
// NV12 fast-path helper (shared with ANSYOLOV12RTOD, ANSYOLOV10RTOD)
NV12PreprocessHelper m_nv12Helper;
// Per-call image geometry for concurrent inference
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
// ── GPU Preprocessing ────────────────────────────────────────────
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(
const cv::Mat& inputImage, ImageMetadata& outMeta);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(
const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata);
// ── Detection pipeline ───────────────────────────────────────────
std::vector<Object> DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id);
std::vector<std::vector<Object>> DetectObjectsBatch(
const std::vector<cv::Mat>& inputImages, const std::string& camera_id);
// ── Task-specific postprocessors (legacy format) ─────────────────
std::vector<Object> PostprocessDetection(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessOBB(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessSegmentation(
std::vector<std::vector<float>>& featureVectors,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessPose(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessClassify(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
// ── End2end postprocessors ───────────────────────────────────────
std::vector<Object> PostprocessDetectionE2E(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessOBBE2E(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessSegE2E(
std::vector<std::vector<float>>& featureVectors,
const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> PostprocessPoseE2E(
std::vector<float>& featureVector,
const std::string& camera_id, const ImageMetadata& meta);
// ── OBB NMS helpers (Prob-IoU based) ─────────────────────────────
struct OrientedBox {
float x, y, width, height, angle;
};
static void getCovarianceComponents(const OrientedBox& box,
float& out1, float& out2, float& out3);
static std::vector<std::vector<float>> batchProbiou(
const std::vector<OrientedBox>& obb1,
const std::vector<OrientedBox>& obb2, float eps = 1e-7f);
static std::vector<int> nmsRotatedImpl(
const std::vector<OrientedBox>& sortedBoxes, float iouThreshold);
static std::vector<int> nmsRotated(
const std::vector<OrientedBox>& boxes,
const std::vector<float>& scores, float iouThreshold);
static std::vector<cv::Point2f> OBBToPoints(const OrientedBox& obb);
// ── TensorRT engine (shared across tasks using same model) ──────
std::shared_ptr<Engine<float>> m_trtEngine = nullptr;
EnginePoolManager<float>::PoolKey m_poolKey; // key for release
bool m_usingSharedPool = false;
// Preprocessing constants: YOLO expects [0,1] normalized input
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
ANSCENTER::Options m_options;
// Filter thresholds
float PROBABILITY_THRESHOLD{ 0.25f };
float NMS_THRESHOLD{ 0.45f };
int TOP_K{ 300 };
// Segmentation constants
int SEG_CHANNELS{ 32 };
int SEG_H{ 160 };
int SEG_W{ 160 };
float SEGMENTATION_THRESHOLD{ 0.5f };
// Pose estimation constants
int NUM_KPS{ 0 };
float KPS_THRESHOLD{ 0.5f };
};
} // namespace ANSCENTER
#endif
+734
View File
@@ -0,0 +1,734 @@
#include "ANSSAM.h"
#include <algorithm>
#include <filesystem>
#include <algorithm>
namespace ANSCENTER {
// public:
bool ANSSAM::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("ANSSAM::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("ANSSAM::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("ANSSAM::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool ANSSAM::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
std::string xmlfile = CreateFilePath(_modelFolder, "anssam.xml");
std::string binaryfile = CreateFilePath(_modelFolder, "anssam.bin");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSSAM::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSSAM::Initialize binary weight", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSSAM::Initialize. Model binary file does not exist", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model file does exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
if (_modelConfig.modelConfThreshold <= 0)_modelConfig.modelConfThreshold = 0.4;
if (_modelConfig.modelMNSThreshold <= 0)_modelConfig.modelMNSThreshold = 0.4;
_isInitialized = Init(_modelFilePath, _modelConfig.modelConfThreshold, _modelConfig.modelMNSThreshold, true);
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSPOSE::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSSAM::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "anssam";
}
std::string xmlFileName = _modelName + ".xml";
std::string binFileName = _modelName + ".bin";
std::string xmlfile = CreateFilePath(_modelFolder, xmlFileName);
std::string binaryfile = CreateFilePath(_modelFolder, binFileName);
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSSAM::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSSAM::Initialize binary weight", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSSAM::Initialize. Model binary file does not exist", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model file does exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
if (modelConfig.modelConfThreshold <= 0)modelConfig.modelConfThreshold = 0.4;
if (modelConfig.modelMNSThreshold <= 0)modelConfig.modelMNSThreshold = 0.4;
labelMap = "SAM";
_isInitialized = Init(_modelFilePath, modelConfig.modelConfThreshold, modelConfig.modelMNSThreshold, true);
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSPOSE::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSSAM::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap)
{
try {
std::lock_guard<std::recursive_mutex> lock(_mutex);
_modelConfig = modelConfig;
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
std::string xmlfile = CreateFilePath(_modelFolder, "anssam.xml");
std::string binaryfile = CreateFilePath(_modelFolder, "anssam.bin");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSSAM::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
if (FileExist(binaryfile)) {
this->_logger.LogDebug("ANSSAM::Initialize binary weight", binaryfile, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSSAM::Initialize. Model binary file does not exist", binaryfile, __FILE__, __LINE__);
return false;
}
}
else {
this->_logger.LogError("ANSPOSE::Initialize. Model file does exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
if (modelConfig.modelConfThreshold <= 0)modelConfig.modelConfThreshold = 0.4;
if (modelConfig.modelMNSThreshold <= 0)modelConfig.modelMNSThreshold = 0.4;
labelMap = "SAM";
_isInitialized = Init(_modelFilePath, modelConfig.modelConfThreshold, modelConfig.modelMNSThreshold, true);
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSPOSE::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSSAM::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> ANSSAM::RunInference(const cv::Mat& input,const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
if (!_licenseValid || !_isInitialized || input.empty() ||
input.cols < 10 || input.rows < 10) {
return {};
}
// Convert to BGR
cv::Mat processedImage = (input.channels() == 1)
? [&] { cv::Mat tmp; cv::cvtColor(input, tmp, cv::COLOR_GRAY2BGR); return tmp; }()
: input;
// Run inference
ov::Tensor input_tensor = Preprocess(processedImage);
if (input_tensor.get_size() == 0) return {};
m_request.set_input_tensor(input_tensor);
m_request.infer();
std::vector<cv::Mat> masks = Postprocess(processedImage);
if (masks.empty()) return {};
// Build output
std::vector<Object> output;
output.reserve(masks.size());
const cv::Rect imageBounds(0, 0, input.cols, input.rows);
for (auto& mask : masks) {
Object obj;
obj.extraInfo = MaskToPolygons(mask, obj.box, obj.polygon);
obj.box &= imageBounds;
if (obj.box.area() == 0) continue;
obj.confidence = 1.0f;
obj.classId = 0;
obj.className = "sam";
obj.cameraId = camera_id;
output.push_back(std::move(obj));
}
if (_trackerEnabled) {
output = ApplyTracking(output, camera_id);
if (_stabilizationEnabled) output = StabilizeDetections(output, camera_id);
}
return output;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSSAM::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
ANSSAM::~ANSSAM() {
Destroy();
}
bool ANSSAM::Destroy()
{
try {
if (FolderExist(_modelFolder)) {
DeleteFolder(_modelFolder);
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSSAM::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
//private:
bool ANSSAM::Init(const std::string& xml_path, float conf, float iou, bool useGpu)
{
m_conf = conf;
m_iou = iou;
if (!std::filesystem::exists(xml_path))
return false;
m_model = m_core.read_model(xml_path);
if (!ParseArgs())
return false;
if (!BuildProcessor())
return false;
m_compiled_model = m_core.compile_model(m_model, useGpu && IsGpuAvaliable(m_core) ? "GPU" : "CPU");
m_request = m_compiled_model.create_infer_request();
return true;
}
void ANSSAM::Infer(const std::string& image_path)
{
try
{
cv::Mat image = cv::imread(image_path);
cv::Mat rendered = Infer(image);
std::string savedir = std::filesystem::current_path().string() + "/results";
if (!std::filesystem::exists(savedir))
std::filesystem::create_directory(savedir);
std::string savepath = savedir + "/" + std::filesystem::path(image_path).filename().string();
cv::imwrite(savepath, rendered);
//slog::info << "result saved in :" << savepath << "\n";
}
catch (const std::exception& e)
{
//slog::info << "Failed to Infer! ec: " << e.what() << '\n';
}
}
cv::Mat ANSSAM::Infer(const cv::Mat& image)
{
cv::Mat processedImg = image.clone();
ov::Tensor input_tensor = Preprocess(processedImg);
assert(input_tensor.get_size() != 0);
m_request.set_input_tensor(input_tensor);
m_request.infer();
std::vector<cv::Mat> result = Postprocess(image);
return Render(image, result);
}
bool ANSSAM::ParseArgs()
{
try
{
model_input_shape = m_model->input().get_shape();
model_output0_shape = m_model->output(0).get_shape();
model_output1_shape = m_model->output(1).get_shape();
//slog::info << "xml input shape:" << model_input_shape << "\n";
//slog::info << "xml output shape 0:" << model_output0_shape << "\n";
//slog::info << "xml output shape 1:" << model_output1_shape << "\n";
// [1, 3, 640, 640]
input_channel = model_input_shape[1];
input_height = model_input_shape[2];
input_width = model_input_shape[3];
this->input_data.resize(input_channel * input_height * input_height);
//slog::info << "model input height:" << input_height << " input width:" << input_width << "\n";
// output0 = [1,37,8400]
// output1 = [1,32,160,160]
mh = model_output1_shape[2];
mw = model_output1_shape[3];
//slog::info << "model output mh:" << mh << " output mw:" << mw << "\n";
}
catch (const std::exception& e)
{
//slog::info << "Failed to Parse Args. " << e.what() << '\n';
return false;
}
return true;
}
void ANSSAM::ScaleBoxes(cv::Mat& box, const cv::Size& oriSize)
{
float oriWidth = static_cast<float>(oriSize.width);
float oriHeight = static_cast<float>(oriSize.height);
float* pxvec = box.ptr<float>(0);
for (int i = 0; i < box.rows; i++) {
pxvec = box.ptr<float>(i);
pxvec[0] -= this->dw;
pxvec[0] = std::clamp(pxvec[0] * this->ratio, 0.f, oriWidth);
pxvec[1] -= this->dh;
pxvec[1] = std::clamp(pxvec[1] * this->ratio, 0.f, oriHeight);
pxvec[2] -= this->dw;
pxvec[2] = std::clamp(pxvec[2] * this->ratio, 0.f, oriWidth);
pxvec[3] -= this->dh;
pxvec[3] = std::clamp(pxvec[3] * this->ratio, 0.f, oriHeight);
}
}
std::vector<cv::Mat> ANSSAM::NMS(cv::Mat& prediction, int max_det)
{
std::vector<cv::Mat> vreMat;
cv::Mat temData = cv::Mat();
prediction = prediction.t(); // [37, 8400] --> [rows:8400, cols:37]
float* pxvec = prediction.ptr<float>(0);
for (int i = 0; i < prediction.rows; i++) {
pxvec = prediction.ptr<float>(i);
if (pxvec[4] > m_conf) {
temData.push_back(prediction.rowRange(i, i + 1).clone());
}
}
if (temData.rows == 0) {
return vreMat;
}
cv::Mat box = temData.colRange(0, 4).clone(); //
cv::Mat cls = temData.colRange(4, 5).clone(); //
cv::Mat mask = temData.colRange(5, temData.cols).clone(); //
cv::Mat j = cv::Mat::zeros(cls.size(), CV_32F);
cv::Mat dst;
cv::hconcat(box, cls, dst); // concat
cv::hconcat(dst, j, dst);
cv::hconcat(dst, mask, dst); // dst = [box class j mask] concat
std::vector<float> scores;
std::vector<cv::Rect> boxes;
pxvec = dst.ptr<float>(0);
for (int i = 0; i < dst.rows; i++) {
pxvec = dst.ptr<float>(i);
boxes.push_back(cv::Rect(pxvec[0], pxvec[1], pxvec[2], pxvec[3]));
scores.push_back(pxvec[4]);
}
std::vector<int> indices;
cv::dnn::NMSBoxes(boxes, scores, m_conf, m_iou, indices);
cv::Mat reMat;
for (int i = 0; i < indices.size() && i < max_det; i++) {
int index = indices[i];
reMat.push_back(dst.rowRange(index, index + 1).clone());
}
box = reMat.colRange(0, 6).clone();
xywh2xyxy(box);
mask = reMat.colRange(6, reMat.cols).clone();
vreMat.push_back(box);
vreMat.push_back(mask);
return vreMat;
}
void ANSSAM::xywh2xyxy(cv::Mat& box)
{
float* pxvec = box.ptr<float>(0);
for (int i = 0; i < box.rows; i++) {
pxvec = box.ptr<float>(i);
float w = pxvec[2];
float h = pxvec[3];
float cx = pxvec[0];
float cy = pxvec[1];
pxvec[0] = cx - w / 2;
pxvec[1] = cy - h / 2;
pxvec[2] = cx + w / 2;
pxvec[3] = cy + h / 2;
}
}
cv::Mat ANSSAM::ConvertToBinary(cv::Mat src) {
cv::Mat gray, binary;
// Check if the image is already in grayscale
if (src.channels() == 3) {
// Convert from BGR to Grayscale
cv::cvtColor(src, gray, cv::COLOR_BGR2GRAY);
}
else if (src.channels() == 1) {
gray = src.clone(); // Use the grayscale image as is
}
else {
std::cerr << "Unsupported number of channels: " << src.channels() << std::endl;
return cv::Mat(); // Return an empty matrix on failure
}
// Convert grayscale to binary
cv::threshold(gray, binary, 127, 255, cv::THRESH_BINARY); // Adjust the threshold as needed
cv::Mat binaryMask;
cv::threshold(binary, binaryMask, 0.5, 255, cv::THRESH_BINARY);
binaryMask.convertTo(binaryMask, CV_8UC1);
return binaryMask;
}
std::string ANSSAM::MaskToPolygons(const cv::Mat& image, cv::Rect& boundingBox, std::vector<cv::Point2f>& polygon) {
try {
// image is boxMask: CV_32FC1 with 0.0 / 1.0 → directly convert
cv::Mat mask8u;
image.convertTo(mask8u, CV_8UC1, 255.0); // 1.0 → 255, 0.0 → 0
if (mask8u.empty() || mask8u.type() != CV_8UC1) {
return "";
}
std::vector<std::vector<cv::Point>> contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(mask8u, contours, hierarchy, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
if (contours.empty()) {
return "";
}
// Select the largest contour
size_t largestContourIdx = 0;
double maxArea = 0.0;
for (size_t i = 0; i < contours.size(); ++i) {
double area = cv::contourArea(contours[i]);
if (area > maxArea) {
maxArea = area;
largestContourIdx = i;
}
}
// Approximate polygon
std::vector<cv::Point> approxPolygon;
cv::approxPolyDP(contours[largestContourIdx], approxPolygon, 3, true);
// Convert to Point2f
polygon.clear();
for (const auto& pt : approxPolygon) {
polygon.push_back(cv::Point2f(pt.x, pt.y));
}
// Calculate bounding box
boundingBox = cv::boundingRect(approxPolygon);
// Convert polygon to string
std::stringstream xCoords, yCoords;
for (size_t i = 0; i < polygon.size(); ++i) {
if (i > 0) {
xCoords << "; ";
yCoords << "; ";
}
xCoords << polygon[i].x;
yCoords << polygon[i].y;
}
std::string formattedString = xCoords.str() + " | " + yCoords.str();
return formattedString;
}
catch (const std::exception& e) {
//slog::info << "Failed to convert mask to polygons: " << e.what() << '\n';
return "";
}
}
cv::Mat ANSSAM::Render(const cv::Mat& image, const std::vector<cv::Mat>& vremat)
{
cv::Mat rendered = image.clone();
for (const auto& mask : vremat) {
auto color = RandomColor();
for (int y = 0; y < mask.rows; y++) {
const float* mp = mask.ptr<float>(y);
uchar* p = rendered.ptr<uchar>(y);
for (int x = 0; x < mask.cols; x++) {
if (mp[x] == 1.0) { // ??
p[0] = cv::saturate_cast<uchar>(p[0] * 0.5 + color[0] * 0.5);
p[1] = cv::saturate_cast<uchar>(p[1] * 0.5 + color[1] * 0.5);
p[2] = cv::saturate_cast<uchar>(p[2] * 0.5 + color[2] * 0.5);
}
p += 3;
}
}
}
return rendered;
}
ov::Tensor ANSSAM::Preprocess(cv::Mat& image)
{
if (!ConvertSize(image)) {
//slog::info << "failed to Convert Size!\n";
return ov::Tensor();
}
if (!ConvertLayout(image)) {
//slog::info << "Failed to Convert Layout!\n";
return ov::Tensor();
}
return BuildTensor();
}
cv::Mat ANSSAM::BuildOutput0()
{
auto* ptr = m_request.get_output_tensor(0).data();
return cv::Mat(model_output0_shape[1], model_output0_shape[2], CV_32F, ptr);
}
cv::Mat ANSSAM::BuildOutput1()
{
auto* ptr = m_request.get_output_tensor(1).data();
return cv::Mat(model_output1_shape[1], model_output1_shape[2] * model_output1_shape[3], CV_32F, ptr);
}
std::vector<cv::Mat> ANSSAM::ProcessMaskNative(const cv::Mat& image, cv::Mat& protos, cv::Mat& masks_in, cv::Mat& bboxes, cv::Size shape)
{
std::vector<cv::Mat> result;
cv::Mat matmulRes = (masks_in * protos).t(); //
cv::Mat maskMat = matmulRes.reshape(bboxes.rows, { mh, mw });
std::vector<cv::Mat> maskChannels;
cv::split(maskMat, maskChannels);
int scale_dw = this->dw / input_width * mw;
int scale_dh = this->dh / input_height * mh;
cv::Rect roi(scale_dw, scale_dh, mw - 2 * scale_dw, mh - 2 * scale_dh);
float* pxvec = bboxes.ptr<float>(0);
for (int i = 0; i < bboxes.rows; i++) {
pxvec = bboxes.ptr<float>(i);
cv::Mat dest, mask;
cv::exp(-maskChannels[i], dest);
dest = 1.0 / (1.0 + dest);
dest = dest(roi);
cv::resize(dest, mask, image.size(), cv::INTER_LINEAR);
cv::Rect roi(pxvec[0], pxvec[1], pxvec[2] - pxvec[0], pxvec[3] - pxvec[1]);
cv::Mat temmask = mask(roi);
cv::Mat boxMask = cv::Mat(image.size(), mask.type(), cv::Scalar(0.0));
float rx = std::max(pxvec[0], 0.0f);
float ry = std::max(pxvec[1], 0.0f);
for (int y = ry, my = 0; my < temmask.rows; y++, my++) {
float* ptemmask = temmask.ptr<float>(my);
float* pboxmask = boxMask.ptr<float>(y);
for (int x = rx, mx = 0; mx < temmask.cols; x++, mx++) {
pboxmask[x] = ptemmask[mx] > 0.5 ? 1.0 : 0.0;
}
}
result.push_back(boxMask);
}
return result;
}
std::vector<cv::Mat> ANSSAM::Postprocess(const cv::Mat& oriImage)
{
cv::Mat prediction = BuildOutput0();
cv::Mat proto = BuildOutput1();
std::vector<cv::Mat> remat = NMS(prediction, 100);
if (remat.size() < 2) {
return std::vector<cv::Mat>();
}
cv::Mat box = remat[0];
cv::Mat mask = remat[1];
ScaleBoxes(box, oriImage.size());
return ProcessMaskNative(oriImage, proto, mask, box, oriImage.size());
}
bool ANSSAM::ConvertSize(cv::Mat& image)
{
float height = static_cast<float>(image.rows);
float width = static_cast<float>(image.cols);
float r = std::min(input_height / height, input_width / width);
int padw = static_cast<int>(std::round(width * r)); //
int padh = static_cast<int>(std::round(height * r));
if ((int)width != padw || (int)height != padh)
cv::resize(image, image, cv::Size(padw, padh));
float _dw = (input_width - padw) / 2.f;
float _dh = (input_height - padh) / 2.f;
int top = static_cast<int>(std::round(_dh - 0.1f));
int bottom = static_cast<int>(std::round(_dh + 0.1f));
int left = static_cast<int>(std::round(_dw - 0.1f));
int right = static_cast<int>(std::round(_dw + 0.1f));
cv::copyMakeBorder(image, image, top, bottom, left, right, cv::BORDER_CONSTANT,
cv::Scalar(114, 114, 114));
this->ratio = 1 / r;
this->dw = _dw;
this->dh = _dh;
return true;
}
bool ANSSAM::ConvertLayout(cv::Mat& image)
{
int row = image.rows;
int col = image.cols;
int channels = image.channels();
if (channels != 3)
return false;
cv::cvtColor(image, image, cv::COLOR_BGR2RGB);
for (int c = 0; c < channels; c++) {
for (int i = 0; i < row; i++) {
for (int j = 0; j < col; j++) {
float pix = image.at<cv::Vec3b>(i, j)[c];
input_data[c * row * col + i * col + j] = pix / 255.0;
}
}
}
return true;
}
ov::Tensor ANSSAM::BuildTensor()
{
ov::Shape shape = { 1, static_cast<unsigned long>(input_channel), static_cast<unsigned long>(input_height), static_cast<unsigned long>(input_width) };
return ov::Tensor(ov::element::f32, shape, input_data.data());;
}
bool ANSSAM::BuildProcessor()
{
try
{
m_ppp = std::make_shared<ov::preprocess::PrePostProcessor>(m_model);
m_ppp->input().tensor()
.set_shape({ 1, input_channel, input_height, input_width })
.set_element_type(ov::element::f32)
.set_layout("NCHW")
.set_color_format(ov::preprocess::ColorFormat::RGB);
m_model = m_ppp->build();
}
catch (const std::exception& e)
{
std::cerr << "Failed to build the model processor!\n" << e.what() << '\n';
return false;
}
return true;
}
bool ANSSAM::IsGpuAvaliable(const ov::Core& core)
{
std::vector<std::string> avaliableDevice = core.get_available_devices();
auto iter = std::find(avaliableDevice.begin(), avaliableDevice.end(), "GPU");
return iter != avaliableDevice.end();
}
cv::Scalar ANSSAM::RandomColor()
{
int b = rand() % 256;
int g = rand() % 256;
int r = rand() % 256;
return cv::Scalar(b, g, r);
}
}
// std::vector<Object> ANSSAM::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> detectionObjects;
// try {
// if (!_licenseValid) {
// this->_logger.LogError("ANSSAM::RunInference", "Invalid License", __FILE__, __LINE__);
// return detectionObjects;
// }
// if (!_isInitialized) {
// this->_logger.LogError("ANSSAM::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return detectionObjects;
// }
// if (input.empty()) return detectionObjects;
// if ((input.cols < 10) || (input.rows < 10)) return detectionObjects;
// // Convert grayscale to 3-channel BGR if needed
// cv::Mat processedImage;
// if (input.channels() == 1) {
// cv::cvtColor(input, processedImage, cv::COLOR_GRAY2BGR);
// }
// else {
// processedImage = input;
// }
// ov::Tensor input_tensor = Preprocess(processedImage);
// assert(input_tensor.get_size() != 0);
// m_request.set_input_tensor(input_tensor);
// m_request.infer();
// std::vector<cv::Mat> result = Postprocess(input);
// for (int i = 0; i < result.size(); i++) {
// Object detectedObject;
// cv::Rect rect;
// std::vector<cv::Point2f> polygon;
// std::string extraInfo;
// extraInfo = MaskToPolygons(result.at(i), rect, polygon);
// detectedObject.box = rect;
// detectedObject.box.x = std::max(detectedObject.box.x, 0);
// detectedObject.box.y = std::max(detectedObject.box.y, 0);
// detectedObject.box.width = std::min(detectedObject.box.width, input.cols - detectedObject.box.x);
// detectedObject.box.height = std::min(detectedObject.box.height, input.rows - detectedObject.box.y);
// detectedObject.polygon = polygon;
// detectedObject.extraInfo = extraInfo;
// detectedObject.confidence = 1.0;
// detectedObject.classId = 0;
// detectedObject.className = "sam";
// detectedObject.cameraId = camera_id;
// detectionObjects.push_back(detectedObject);
// }
////EnqueueDetection(detectionObjects,camera_id);
// return detectionObjects;
// }
// catch (std::exception& e) {
// detectionObjects.clear();
// this->_logger.LogFatal("ANSSAM::RunInference", e.what(), __FILE__, __LINE__);
// return detectionObjects;
// }
// }
+71
View File
@@ -0,0 +1,71 @@
#ifndef ANSSAM_H
#define ANSSAM_H
#pragma once
#include "ANSEngineCommon.h"
#include <string>
#include <opencv2/opencv.hpp>
#include <openvino/openvino.hpp>
#include <array>
namespace ANSCENTER
{
class ANSENGINE_API ANSSAM: public ANSODBase
{
protected:
std::string _modelFilePath;
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSSAM();
private:
bool Init(const std::string& xml_path, float conf, float iou, bool useGpu);
void Infer(const std::string& image_path);
cv::Mat Infer(const cv::Mat& image);
std::vector<cv::Mat> Postprocess(const cv::Mat& oriImage);
cv::Mat BuildOutput0();
cv::Mat BuildOutput1();
void ScaleBoxes(cv::Mat& box, const cv::Size& oriSize);
std::vector<cv::Mat> ProcessMaskNative(const cv::Mat& oriImage, cv::Mat& protos, cv::Mat& masks_in, cv::Mat& bboxes, cv::Size shape);
std::vector<cv::Mat> NMS(cv::Mat& prediction, int max_det = 300);
void xywh2xyxy(cv::Mat& box);
ov::Tensor Preprocess(cv::Mat& image);
bool ConvertSize(cv::Mat& image);
bool ConvertLayout(cv::Mat& image);
ov::Tensor BuildTensor();
bool ParseArgs();
bool BuildProcessor();
bool IsGpuAvaliable(const ov::Core& core);
cv::Scalar RandomColor();
cv::Mat ConvertToBinary(cv::Mat src);
std::string MaskToPolygons(const cv::Mat& image, cv::Rect& boundingBox, std::vector<cv::Point2f>& polygon);
cv::Mat Render(const cv::Mat& image, const std::vector<cv::Mat>& vremat);
private:
std::shared_ptr<ov::Model> m_model;
ov::CompiledModel m_compiled_model;
ov::Core m_core;
ov::InferRequest m_request;
std::shared_ptr<ov::preprocess::PrePostProcessor> m_ppp;
float m_conf;
float m_iou;
std::vector<float> input_data;
int input_width = 0;
int input_height = 0;
int input_channel = 3;
ov::Shape model_input_shape;
ov::Shape model_output0_shape;
ov::Shape model_output1_shape;
float ratio = 1.0f;
float dw = 0.f;
float dh = 0.f;
int mw = 160;
int mh = 160;
cv::Mat m_image;
};
}
#endif
File diff suppressed because it is too large Load Diff
+122
View File
@@ -0,0 +1,122 @@
#ifndef ANSSAM3_H
#define ANSSAM3_H
#pragma once
#include "ANSEngineCommon.h"
#include "ANSCLIPTokenizer.h"
#include "engine.h"
#include "EPLoader.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
#include <unordered_map>
#include <unordered_set>
namespace ANSCENTER
{
/// SAM3 (Segment Anything Model 3) - text-prompted instance segmentation
/// Full TRT architecture:
/// 1) Image encoder (TRT FP32) — produces backbone features + position encodings
/// 2) Language encoder (TRT FP16) — produces text attention mask + text features
/// 3) Decoder (TRT FP16) — combines image + language features → boxes, scores, masks
class ANSENGINE_API ANSSAM3 : public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath, const std::string& modelZipPassword,
std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder, std::string& labelMap) override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
bool SetPrompt(const std::vector<int64_t>& inputIds, const std::vector<int64_t>& attentionMask) override;
bool SetPrompt(const std::string& text) override;
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSSAM3();
/// Precision mode for TRT engine builds
enum class TrtPrecision { FP16, BF16, FP32 };
private:
// -----------------------------------------------------------------
// TRTBundle — one TensorRT sub-model (engine + context + GPU buffers)
// -----------------------------------------------------------------
struct TRTBundle
{
std::unique_ptr<nvinfer1::IRuntime> runtime;
std::unique_ptr<nvinfer1::ICudaEngine> engine;
std::unique_ptr<nvinfer1::IExecutionContext> context;
std::vector<void*> gpuBuffers; // I/O buffers (GPU or host), one per tensor
std::vector<size_t> gpuBufferSizes; // bytes per buffer
std::unordered_set<int> hostBufferIdx; // indices of host-allocated buffers (shape tensors)
std::unordered_map<std::string, int> nameToIdx; // tensor name → buffer index
void destroy();
};
std::string _modelFilePath;
bool _modelLoadValid = false;
bool _fp16 = false;
// TRT engine bundles
TRTBundle m_imgEncoder;
TRTBundle m_langEncoder;
TRTBundle m_decoder;
cudaStream_t m_cudaStream = nullptr;
// NV12 fast-path helper (fused NV12→RGB CHW directly into TRT buffer)
NV12PreprocessHelper m_nv12Helper;
// Cached language encoder outputs (GPU-resident, set by SetPrompt)
void* m_cachedLangMask = nullptr; // bool/int32 [1, 32]
size_t m_cachedLangMaskBytes = 0;
void* m_cachedLangFeats = nullptr; // float32 [32, 1, 256]
size_t m_cachedLangFeatsBytes = 0;
// Model constants
int m_inputSize = 1008; // image spatial size (H=W) — 3-model split uses 1008
int m_tokenLength = 32;
bool m_promptSet = false;
// Segmentation threshold
float m_segThreshold = 0.5f;
// Tokenizer
std::unique_ptr<ANSCLIPTokenizer> m_tokenizer;
// --- Internal methods ---
bool BuildAndLoadEngine(TRTBundle& bundle, const std::string& onnxPath,
const std::string& label, TrtPrecision precision = TrtPrecision::FP16);
bool LoadTRTEngineBundle(TRTBundle& bundle, const std::string& enginePath, const std::string& label);
/// Pre-build any uncached TRT engines one at a time (avoids GPU OOM during build)
bool EnsureEnginesBuilt(const std::string& imgOnnx, const std::string& langOnnx, const std::string& decOnnx);
/// Generate engine cache filename: <stem>.engine.<GPUName>.<fp16|bf16|fp32>
std::string EngineFileName(const std::string& onnxPath, TrtPrecision precision = TrtPrecision::FP16) const;
std::vector<Object> Detect(const cv::Mat& input, const std::string& camera_id);
std::vector<Object> PostprocessInstances(
const float* boxesData, int numBoxes,
const float* scoresData,
const bool* masksData,
int maskH, int maskW,
int origWidth, int origHeight,
const std::string& camera_id);
static size_t DataTypeSize(nvinfer1::DataType dtype);
Logger m_trtLogger;
};
}
#endif
+988
View File
@@ -0,0 +1,988 @@
#include "ANSTENSORRTCL.h"
#include "Utility.h"
#include <opencv2/cudaimgproc.hpp>
#include <future>
namespace ANSCENTER
{
bool TENSORRTCL::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (!FileExist(_modelFilePath)) {
this->_logger.LogFatal("TENSORRTCL::OptimizeModel", "Raw model file path does not exist", __FILE__, __LINE__);
return false;
}
try {
_fp16 = fp16;
optimizedModelFolder = GetParentFolder(_modelFilePath);
// Check if the engine already exists to avoid reinitializing
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = optimizedModelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// Build the TensorRT engine
auto succ = m_trtEngine->buildWithRetry(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE);
if (!succ) {
const std::string errMsg =
"Error: Unable to build the TensorRT engine. "
"Try increasing TensorRT log severity to kVERBOSE.";
this->_logger.LogError("TENSORRTCL::OptimizeModel", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
_modelLoadValid = true;
return true;
}
catch (const std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::OptimizeModel", e.what(), __FILE__, __LINE__);
optimizedModelFolder.clear();
return false;
}
}
bool TENSORRTCL::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
_fp16 = true; // Load Model from Here
// Load Model from Here
TOP_K = 100;
SEG_CHANNELS = 32;
PROBABILITY_THRESHOLD = 0.3;
NMS_THRESHOLD = 0.65f;
SEGMENTATION_THRESHOLD = 0.5f;
SEG_H = 160;
SEG_W = 160;
NUM_KPS = _modelConfig.numKPS;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
SEG_CHANNELS = 32; // For segmentation
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = _modelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("TENSORRTCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("TENSORRTCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Load the TensorRT engine file
if (this->_loadEngineOnCreation) {
auto succ = m_trtEngine->buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
if (!succ) {
const std::string errMsg = "Error: Unable to load TensorRT engine weights into memory. " + _modelFilePath;
this->_logger.LogError("TENSORRTCL::Initialize", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
}
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool TENSORRTCL::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".onnx";
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
_fp16 = true; // Load Model from Here
// Load Model from Here
TOP_K = 100;
SEG_CHANNELS = 32;
PROBABILITY_THRESHOLD = 0.3;
NMS_THRESHOLD = 0.65f;
SEGMENTATION_THRESHOLD = 0.5f;
SEG_H = 160;
SEG_W = 160;
NUM_KPS = _modelConfig.numKPS;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
SEG_CHANNELS = 32; // For segmentation
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = _modelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
_classFilePath = CreateFilePath(_modelFolder, className);
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("TENSORRTCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("TENSORRTCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load the TensorRT engine file
if (this->_loadEngineOnCreation) {
auto succ = m_trtEngine->buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
if (!succ) {
const std::string errMsg = "Error: Unable to load TensorRT engine weights into memory. " + _modelFilePath;
this->_logger.LogError("TENSORRTCL::Initialize", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
}
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool TENSORRTCL::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
const bool engineAlreadyLoaded = _modelLoadValid && _isInitialized && m_trtEngine != nullptr;
_modelLoadValid = false;
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::CLASSIFICATION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 224;
_modelConfig.inpWidth = 224;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
_fp16 = true; // Load Model from Here
// Load Model from Here
TOP_K = 100;
SEG_CHANNELS = 32;
PROBABILITY_THRESHOLD = 0.3;
NMS_THRESHOLD = 0.65f;
SEGMENTATION_THRESHOLD = 0.5f;
SEG_H = 160;
SEG_W = 160;
NUM_KPS = _modelConfig.numKPS;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
SEG_CHANNELS = 32; // For segmentation
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = _modelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("TENSORRTCL::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("TENSORRTCL::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load the TensorRT engine file
if (this->_loadEngineOnCreation && !engineAlreadyLoaded) {
auto succ = m_trtEngine->buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
if (!succ) {
const std::string errMsg = "Error: Unable to load TensorRT engine weights into memory. " + _modelFilePath;
this->_logger.LogError("TENSORRTCL::Initialize", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
}
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> TENSORRTCL::RunInference(const cv::Mat& inputImgBGR) {
return RunInference(inputImgBGR, "CustomCam");
}
std::vector<Object> TENSORRTCL::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)
{
// Validate state under brief lock
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!_modelLoadValid) {
_logger.LogError("TENSORRTCL::RunInference",
"Cannot load the TensorRT model. Please check if it exists",
__FILE__, __LINE__);
return {};
}
if (!_licenseValid) {
_logger.LogError("TENSORRTCL::RunInference",
"Runtime license is not valid or expired. Please contact ANSCENTER",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("TENSORRTCL::RunInference",
"Model is not initialized",
__FILE__, __LINE__);
return {};
}
if (inputImgBGR.empty() || inputImgBGR.cols < 5 || inputImgBGR.rows < 5) {
return {};
}
}
try {
return DetectObjects(inputImgBGR, camera_id);
}
catch (const std::exception& e) {
_logger.LogFatal("TENSORRTCL::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<std::vector<Object>> TENSORRTCL::RunInferencesBatch(const std::vector<cv::Mat>& inputs, const std::string& camera_id) {
// Validate state under brief lock
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!_modelLoadValid) {
this->_logger.LogFatal("TENSORRTCL::RunInferencesBatch",
"Cannot load the TensorRT model. Please check if it exists", __FILE__, __LINE__);
return {};
}
if (!_licenseValid) {
this->_logger.LogFatal("TENSORRTCL::RunInferencesBatch",
"Runtime license is not valid or expired. Please contact ANSCENTER", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
this->_logger.LogFatal("TENSORRTCL::RunInferencesBatch",
"Engine not initialized", __FILE__, __LINE__);
return {};
}
if (inputs.empty()) return {};
}
try {
return DetectObjectsBatch(inputs, camera_id);
}
catch (const std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::RunInferencesBatch", e.what(), __FILE__, __LINE__);
return {};
}
};
TENSORRTCL::~TENSORRTCL() {
try {
Destroy();
}
catch (std::exception& e) {
this->_logger.LogError("TENSORRTCL::~TENSORRTCL()", e.what(), __FILE__, __LINE__);
}
}
bool TENSORRTCL::Destroy() {
try {
m_trtEngine.reset(); // Releases the current engine and sets m_trtEngine to nullptr.
return true;
}
catch (std::exception& e) {
this->_logger.LogError("TENSORRTCL::~TENSORRTCL()", e.what(), __FILE__, __LINE__);
return false;
}
}
// private
std::vector<Object> TENSORRTCL::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {
try {
// --- 1. Set GPU device context ---
if (m_trtEngine) {
m_trtEngine->setDeviceContext();
}
// --- 1b. CUDA context health check ---
if (!m_nv12Helper.isCudaContextHealthy(_logger, "TENSORRTCL")) {
return {};
}
// --- 2. Preprocess under lock ---
// Try NV12 fast path first (classification: direct resize, no letterbox).
ImageMetadata meta;
std::vector<std::vector<cv::cuda::GpuMat>> input;
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
const int inferenceGpu = m_trtEngine ? m_trtEngine->getPreferredDeviceIndex() : 0;
const auto& inputDims = m_trtEngine->getInputDims();
const int inputW = inputDims[0].d[2];
const int inputH = inputDims[0].d[1];
auto nv12 = m_nv12Helper.tryNV12(inputImage, inferenceGpu, inputW, inputH,
NV12PreprocessHelper::classificationLauncher(),
_logger, "TENSORRTCL");
if (nv12.succeeded) {
meta.imgWidth = nv12.metaWidth;
meta.imgHeight = nv12.metaHeight;
meta.ratio = 1.f; // classification: no letterbox
input = {{ std::move(nv12.gpuRGB) }};
}
else if (nv12.useBgrFullRes) {
input = Preprocess(nv12.bgrFullResImg, meta);
}
if (input.empty()) {
input = Preprocess(inputImage, meta);
}
m_nv12Helper.tickInference();
}
if (input.empty()) return {};
// Phase 2: Inference — mutex released; pool dispatches to idle GPU slot
std::vector<std::vector<std::vector<float>>> featureVectors;
auto succ = m_trtEngine->runInference(input, featureVectors);
if (!succ) {
this->_logger.LogFatal("TENSORRTCL::DetectObjects", "Error running inference", __FILE__, __LINE__);
return {};
}
// Phase 3: Postprocess under brief lock
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<float> featureVector;
Engine<float>::transformOutput(featureVectors, featureVector);
return Postprocess(featureVector, camera_id, meta);
}
catch (std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::DetectObjects", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<std::vector<cv::cuda::GpuMat>> TENSORRTCL::Preprocess(const cv::Mat& inputImage, ImageMetadata& outMeta) {
try {
if (!_licenseValid) {
this->_logger.LogFatal("TENSORRTCL::Preprocess", "Invalid license", __FILE__, __LINE__);
return {};
}
if (inputImage.empty()) {
this->_logger.LogFatal("TENSORRTCL::Preprocess", "Input image is empty", __FILE__, __LINE__);
return {};
}
if ((inputImage.cols < 5) || (inputImage.rows < 5)) {
this->_logger.LogFatal("TENSORRTCL::Preprocess",
"Input image is too small (Width: " + std::to_string(inputImage.cols) +
", Height: " + std::to_string(inputImage.rows) + ")",
__FILE__, __LINE__);
return {};
}
// Populate the input vectors
const auto& inputDims = m_trtEngine->getInputDims();
const int inputH = inputDims[0].d[1];
const int inputW = inputDims[0].d[2];
// Upload the image to GPU memory
cv::cuda::Stream stream; // Create a custom stream
cv::cuda::GpuMat img;
if (inputImage.channels() == 1) {
// Convert grayscale to 3-channel BGR before uploading
cv::Mat img3Channel;
cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);
img.upload(img3Channel, stream);
}
else {
img.upload(inputImage, stream);
}
// Convert BGR to RGB
cv::cuda::GpuMat imgRGB;
cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);
// These parameters will be used in the post-processing stage
outMeta.imgHeight = imgRGB.rows;
outMeta.imgWidth = imgRGB.cols;
if (outMeta.imgHeight <= 0 || outMeta.imgWidth <= 0) {
_logger.LogFatal("TENSORRTCL::Preprocess", "Image height or width is zero", __FILE__, __LINE__);
return {};
}
if (outMeta.imgHeight > 0 && outMeta.imgWidth > 0) {
outMeta.ratio = 1.f;
cv::cuda::GpuMat resized = imgRGB;
// Classification: direct resize (no letterbox padding) — matches ANSONNXCL
// Must use explicit stream to avoid conflict with CUDA Graph capture on null stream
if (resized.rows != inputDims[0].d[1] || resized.cols != inputDims[0].d[2]) {
cv::cuda::resize(imgRGB, resized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR, stream);
}
// Wait for all GPU ops to complete before returning GpuMats
stream.waitForCompletion();
// Convert to format expected by our inference engine
std::vector<cv::cuda::GpuMat> input{ std::move(resized) };
std::vector<std::vector<cv::cuda::GpuMat>> inputs{ std::move(input) };
return inputs;
}
else {
this->_logger.LogFatal("TENSORRTCL::Preprocess",
"Image height or width is zero after processing (Width: " + std::to_string(outMeta.imgWidth) +
", Height: " + std::to_string(outMeta.imgHeight) + ")",
__FILE__, __LINE__);
return {};
}
}
catch (const std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::Preprocess", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<Object> TENSORRTCL::Postprocess(std::vector<float>& featureVector, const std::string& camera_id, const ImageMetadata& meta) {
std::vector<Object> outputs;
try {
// Check if output is already a probability distribution (sums to ~1.0).
// Some models include a Softmax layer; applying softmax again would
// flatten the distribution and cause wrong classifications.
float rawSum = 0.f;
bool allNonNeg = true;
for (const auto& v : featureVector) {
rawSum += v;
if (v < 0.f) allNonNeg = false;
}
const bool alreadyNormalized = (allNonNeg && rawSum > 0.9f && rawSum < 1.1f);
if (!alreadyNormalized) {
// Raw logits — apply softmax
float maxLogit = *std::max_element(featureVector.begin(), featureVector.end());
float sumExp = 0.f;
for (auto& v : featureVector) {
v = std::exp(v - maxLogit);
sumExp += v;
}
for (auto& v : featureVector)
v /= sumExp;
}
auto max_idx = std::max_element(featureVector.begin(), featureVector.end());
int class_id = static_cast<int>(std::distance(featureVector.begin(), max_idx));
float score = *max_idx;
int classNameSize = _classes.size();
Object clsResult;
clsResult.classId = class_id;
if (!_classes.empty()) {
if (clsResult.classId < classNameSize) {
clsResult.className = _classes[clsResult.classId];
}
else {
clsResult.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
}
}
else {
clsResult.className = "Unknown"; // Fallback if _classes is empty
}
clsResult.confidence = score;
if (meta.imgWidth > 20 && meta.imgHeight > 20) {
clsResult.box = cv::Rect(10, 10, meta.imgWidth - 20, meta.imgHeight - 20);
}
else {
clsResult.box = cv::Rect(0, 0, meta.imgWidth, meta.imgHeight);
}
clsResult.polygon = ANSUtilityHelper::RectToNormalizedPolygon(clsResult.box, meta.imgWidth, meta.imgHeight);
clsResult.cameraId = camera_id;
outputs.push_back(clsResult);
return outputs;
//EnqueueDetection(objects, camera_id);
}
catch (std::exception& e) {
this->_logger.LogFatal("TENSORRTCL::Postproces", e.what(), __FILE__, __LINE__);
return outputs;
}
}
std::vector<std::vector<Object>> TENSORRTCL::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id)
{
// Validate under brief lock
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (inputImages.empty()) {
_logger.LogFatal("TENSORRTCL::DetectObjectsBatch",
"Empty input images vector", __FILE__, __LINE__);
return {};
}
}
// Auto-split if batch exceeds engine capacity
const int maxBatch = m_options.maxBatchSize > 0 ? m_options.maxBatchSize : 1;
if (static_cast<int>(inputImages.size()) > maxBatch) {
const size_t numImages = inputImages.size();
std::vector<std::vector<Object>> allResults;
allResults.reserve(numImages);
// Process chunks sequentially to avoid GPU contention on the same engine
for (size_t start = 0; start < numImages; start += static_cast<size_t>(maxBatch)) {
const size_t end = std::min(start + static_cast<size_t>(maxBatch), numImages);
std::vector<cv::Mat> chunk(inputImages.begin() + start, inputImages.begin() + end);
auto chunkResults = DetectObjectsBatch(chunk, camera_id);
if (chunkResults.size() == chunk.size()) {
for (auto& r : chunkResults) allResults.push_back(std::move(r));
}
else {
// Chunk failed or returned wrong size — pad with empty results
_logger.LogError("TENSORRTCL::DetectObjectsBatch",
"Chunk returned " + std::to_string(chunkResults.size()) +
" results, expected " + std::to_string(chunk.size()) +
". Padding with empty results.", __FILE__, __LINE__);
for (auto& r : chunkResults) allResults.push_back(std::move(r));
for (size_t pad = chunkResults.size(); pad < chunk.size(); ++pad) {
allResults.push_back({});
}
}
}
return allResults;
}
_logger.LogDebug("TENSORRTCL::DetectObjectsBatch",
"Processing batch of " + std::to_string(inputImages.size()) + " images",
__FILE__, __LINE__);
// Phase 1: Preprocess under brief lock
BatchMetadata metadata;
std::vector<std::vector<cv::cuda::GpuMat>> inputs;
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
inputs = PreprocessBatch(inputImages, metadata);
}
if (inputs.empty() || inputs[0].empty()) {
_logger.LogFatal("TENSORRTCL::DetectObjectsBatch",
"Preprocessing failed", __FILE__, __LINE__);
return {};
}
// Phase 2: Inference — mutex released; pool dispatches to idle GPU slot
std::vector<std::vector<std::vector<float>>> featureVectors;
bool succ = m_trtEngine->runInference(inputs, featureVectors);
if (!succ) {
_logger.LogFatal("TENSORRTCL::DetectObjectsBatch",
"Error running batch inference", __FILE__, __LINE__);
return {};
}
// Phase 3: Parallel postprocessing
const size_t numBatch = featureVectors.size();
std::vector<std::vector<Object>> batchDetections(numBatch);
std::vector<std::future<std::vector<Object>>> postFutures;
postFutures.reserve(numBatch);
for (size_t batchIdx = 0; batchIdx < numBatch; ++batchIdx) {
const auto& batchOutput = featureVectors[batchIdx];
std::vector<float> fv = batchOutput.empty() ? std::vector<float>{} : batchOutput[0];
postFutures.push_back(std::async(std::launch::async,
[this, fv = std::move(fv), cid = camera_id, idx = batchIdx, &metadata]() mutable {
return PostprocessBatch(fv, cid, idx, metadata);
}));
}
for (size_t i = 0; i < numBatch; ++i)
batchDetections[i] = postFutures[i].get();
_logger.LogDebug("TENSORRTCL::DetectObjectsBatch",
"Batch processing complete. Images: " + std::to_string(numBatch),
__FILE__, __LINE__);
return batchDetections;
}
std::vector<std::vector<cv::cuda::GpuMat>> TENSORRTCL::PreprocessBatch(const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata)
{
try {
// Validate license
if (!_licenseValid) {
_logger.LogFatal("TENSORRTCL::PreprocessBatch",
"Invalid license", __FILE__, __LINE__);
return {};
}
// Validate input
if (inputImages.empty()) {
_logger.LogFatal("TENSORRTCL::PreprocessBatch",
"Input images vector is empty", __FILE__, __LINE__);
return {};
}
size_t batchSize = inputImages.size();
// Get model input dimensions
const auto& inputDims = m_trtEngine->getInputDims();
const int inputH = inputDims[0].d[1];
const int inputW = inputDims[0].d[2];
_logger.LogDebug("TENSORRTCL::PreprocessBatch",
"Preprocessing " + std::to_string(batchSize) + " images to " +
std::to_string(inputW) + "x" + std::to_string(inputH),
__FILE__, __LINE__);
// Create CUDA stream for async operations
cv::cuda::Stream stream;
// Store ALL images in a SINGLE batch vector
std::vector<cv::cuda::GpuMat> batchedImages;
batchedImages.reserve(batchSize);
// Store image dimensions for postprocessing
outMetadata.imgHeights.clear();
outMetadata.imgWidths.clear();
outMetadata.ratios.clear();
outMetadata.imgHeights.reserve(batchSize);
outMetadata.imgWidths.reserve(batchSize);
outMetadata.ratios.reserve(batchSize);
// Process each image
for (size_t i = 0; i < batchSize; ++i) {
const cv::Mat& inputImage = inputImages[i];
// Validate individual image
if (inputImage.empty()) {
_logger.LogFatal("TENSORRTCL::PreprocessBatch",
"Input image at index " + std::to_string(i) + " is empty",
__FILE__, __LINE__);
return {};
}
if (inputImage.cols < 5 || inputImage.rows < 5) {
_logger.LogFatal("TENSORRTCL::PreprocessBatch",
"Image at index " + std::to_string(i) +
" is too small (Width: " + std::to_string(inputImage.cols) +
", Height: " + std::to_string(inputImage.rows) + ")",
__FILE__, __LINE__);
return {};
}
// Upload to GPU
cv::cuda::GpuMat img;
if (inputImage.channels() == 1) {
// Convert grayscale to BGR
cv::Mat img3Channel;
cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);
img.upload(img3Channel, stream);
}
else {
img.upload(inputImage, stream);
}
// Convert BGR to RGB
cv::cuda::GpuMat imgRGB;
cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);
// Store original dimensions
int imgHeight = imgRGB.rows;
int imgWidth = imgRGB.cols;
if (imgHeight <= 0 || imgWidth <= 0) {
_logger.LogFatal("TENSORRTCL::PreprocessBatch",
"Image at index " + std::to_string(i) + " has zero height or width",
__FILE__, __LINE__);
return {};
}
outMetadata.imgHeights.push_back(imgHeight);
outMetadata.imgWidths.push_back(imgWidth);
// Calculate resize ratio
float ratio = 1.f / std::min(
inputDims[0].d[2] / static_cast<float>(imgRGB.cols),
inputDims[0].d[1] / static_cast<float>(imgRGB.rows)
);
outMetadata.ratios.push_back(ratio);
// Resize maintaining aspect ratio with padding
cv::cuda::GpuMat resized;
if (imgRGB.rows != inputDims[0].d[1] || imgRGB.cols != inputDims[0].d[2]) {
resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(
imgRGB, inputDims[0].d[1], inputDims[0].d[2]
);
}
else {
resized = imgRGB;
}
// Add to batch
batchedImages.push_back(std::move(resized));
}
// Wait for all GPU operations to complete
stream.waitForCompletion();
// Return as single batched input
std::vector<std::vector<cv::cuda::GpuMat>> result;
result.push_back(std::move(batchedImages));
return result;
}
catch (const std::exception& e) {
_logger.LogFatal("TENSORRTCL::PreprocessBatch",
e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<Object> TENSORRTCL::PostprocessBatch(std::vector<float>& featureVector, const std::string& camera_id, size_t batchIdx, const BatchMetadata& metadata)
{
std::vector<Object> outputs;
try {
// Validate batch index
if (batchIdx >= metadata.imgHeights.size() ||
batchIdx >= metadata.imgWidths.size()) {
_logger.LogFatal("TENSORRTCL::PostprocessBatch",
"Batch index " + std::to_string(batchIdx) +
" out of range (stored " + std::to_string(metadata.imgHeights.size()) + " images)",
__FILE__, __LINE__);
return outputs;
}
// Validate feature vector
if (featureVector.empty()) {
_logger.LogFatal("TENSORRTCL::PostprocessBatch",
"Feature vector is empty for batch index " + std::to_string(batchIdx),
__FILE__, __LINE__);
return outputs;
}
// Get image dimensions for this batch index
int imgHeight = metadata.imgHeights[batchIdx];
int imgWidth = metadata.imgWidths[batchIdx];
// Normalize if raw logits (same logic as single-image Postprocess)
float rawSum = 0.f;
bool allNonNeg = true;
for (const auto& v : featureVector) {
rawSum += v;
if (v < 0.f) allNonNeg = false;
}
const bool alreadyNorm = (allNonNeg && rawSum > 0.9f && rawSum < 1.1f);
if (!alreadyNorm) {
float maxLogit = *std::max_element(featureVector.begin(), featureVector.end());
float sumExp = 0.f;
for (auto& v : featureVector) {
v = std::exp(v - maxLogit);
sumExp += v;
}
for (auto& v : featureVector) v /= sumExp;
}
// Find max element (classification result)
auto max_idx = std::max_element(featureVector.begin(), featureVector.end());
if (max_idx == featureVector.end()) {
_logger.LogFatal("TENSORRTCL::PostprocessBatch",
"Failed to find max element in feature vector for batch index " +
std::to_string(batchIdx),
__FILE__, __LINE__);
return outputs;
}
int class_id = static_cast<int>(std::distance(featureVector.begin(), max_idx));
float score = *max_idx;
// Create object result
Object clsResult;
clsResult.classId = class_id;
// Get class name
int classNameSize = static_cast<int>(_classes.size());
if (!_classes.empty()) {
if (class_id >= 0 && class_id < classNameSize) {
clsResult.className = _classes[class_id];
}
else {
clsResult.className = _classes[classNameSize - 1];
}
}
else {
clsResult.className = "Unknown";
}
clsResult.confidence = score;
// Create bounding box with margins
if (imgWidth > 20 && imgHeight > 20) {
clsResult.box = cv::Rect(10, 10, imgWidth - 20, imgHeight - 20);
}
else {
clsResult.box = cv::Rect(0, 0, imgWidth, imgHeight);
}
// Convert to normalized polygon
clsResult.polygon = ANSUtilityHelper::RectToNormalizedPolygon(
clsResult.box, imgWidth, imgHeight
);
clsResult.cameraId = camera_id;
outputs.push_back(std::move(clsResult));
return outputs;
}
catch (const std::exception& e) {
_logger.LogFatal("TENSORRTCL::PostprocessBatch",
"Error for batch index " + std::to_string(batchIdx) + ": " + e.what(),
__FILE__, __LINE__);
return outputs;
}
}
}
+81
View File
@@ -0,0 +1,81 @@
#ifndef ANSTENSORRTCL_H
#define ANSTENSORRTCL_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
namespace ANSCENTER {
// TensorRT class
class ANSENGINE_API TENSORRTCL :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
virtual std::vector<std::vector<Object>> RunInferencesBatch(const std::vector<cv::Mat>& inputs, const std::string& camera_id)override;
bool Destroy();
~TENSORRTCL();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
private:
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
// Preprocess the input
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage, ImageMetadata& outMeta);
// Postprocess the output
std::vector<Object> Postprocess(std::vector<float>& featureVector, const std::string& camera_id, const ImageMetadata& meta);
std::vector<Object> DetectObjects(const cv::Mat& inputImage, const std::string& camera_id);
std::vector<std::vector<Object>> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata);
std::vector<Object> PostprocessBatch(std::vector<float>& featureVector, const std::string& camera_id, size_t batchIdx, const BatchMetadata& metadata);
std::unique_ptr<Engine<float>> m_trtEngine = nullptr;
// Used for image preprocessing
// YoloV8 model expects values between [0.f, 1.f] so we use the following params
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
ANSCENTER::Options m_options;
int m_maxSlotsPerGpu{1}; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
// NV12 fast-path helper (classification uses direct resize, not letterbox)
NV12PreprocessHelper m_nv12Helper;
// Filter thresholds
float PROBABILITY_THRESHOLD;
float NMS_THRESHOLD;
int TOP_K;
// Segmentation constants
int SEG_CHANNELS;
int SEG_H;
int SEG_W;
float SEGMENTATION_THRESHOLD;
// Pose estimation constant
int NUM_KPS;
float KPS_THRESHOLD;
};
}
#endif
+97
View File
@@ -0,0 +1,97 @@
#ifndef ANSTENSORRTOD_H
#define ANSTENSORRTOD_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
namespace ANSCENTER {
// TensorRT class
class ANSENGINE_API TENSORRTOD :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
virtual std::vector<std::vector<Object>> RunInferencesBatch(const std::vector<cv::Mat>& inputs, const std::string& camera_id) override;
bool Destroy();
~TENSORRTOD();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
bool _isFixedBatch{ false };
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
// NV12 fast-path helper (shared with ANSRTYOLO, ANSYOLOV10RTOD, ANSYOLOV12RTOD)
NV12PreprocessHelper m_nv12Helper;
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
// Per-call image geometry — passed between Preprocess and Postprocess
// instead of shared member variables, enabling concurrent inference.
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
private:
// Preprocess the input; fills outMeta with per-call image geometry
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage,
ImageMetadata& outMeta);
// Postprocess the output
std::vector<Object> Postprocess(std::vector<float>& featureVector,
const std::string& camera_id,
const ImageMetadata& meta);
// Postprocess the output for segmentation model
std::vector<Object> PostProcessSegmentation(std::vector<std::vector<float>>& featureVectors,
const std::string& camera_id,
const ImageMetadata& meta);
// Postprocess the output for pose model
std::vector<Object> PostProcessPose(std::vector<float>& featureVector,
const std::string& camera_id,
const ImageMetadata& meta);
std::vector<Object> DetectObjects(const cv::Mat& inputImage, const std::string& camera_id);
// Correct batch detection function signature
std::vector<std::vector<Object>> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,const std::string& camera_id);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(const std::vector<cv::Mat>& inputImages,BatchMetadata& outMetadata); // Return metadata by reference
std::vector<Object> PostprocessBatch(std::vector<float>& featureVector,const std::string& camera_id,size_t batchIdx,const BatchMetadata& metadata); // Accept metadata as parameter
std::unique_ptr<Engine<float>> m_trtEngine = nullptr;
// Used for image preprocessing
// YoloV8 model expects values between [0.f, 1.f] so we use the following params
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
ANSCENTER::Options m_options;
// Filter thresholds
float PROBABILITY_THRESHOLD;
float NMS_THRESHOLD;
int TOP_K;
// Segmentation constants
int SEG_CHANNELS;
int SEG_H;
int SEG_W;
float SEGMENTATION_THRESHOLD;
// Pose estimation constant
int NUM_KPS;
float KPS_THRESHOLD;
};
}
#endif
File diff suppressed because it is too large Load Diff
+87
View File
@@ -0,0 +1,87 @@
#ifndef ANSTENSORRTPOSE_H
#define ANSTENSORRTPOSE_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
namespace ANSCENTER {
// TensorRT class
class ANSENGINE_API ANSTENSORRTPOSE :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
virtual std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs, const std::string& camera_id) override;
bool Destroy();
~ANSTENSORRTPOSE();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
// NV12 fast-path helper
NV12PreprocessHelper m_nv12Helper;
private:
// Per-call batch metadata — passed by out-param to avoid shared state
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
// Per-call single-image metadata
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
// Preprocess the input; fills outMeta with per-call image geometry
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage,
ImageMetadata& outMeta);
// Postprocess the output for pose model
std::vector<Object> PostProcessPose(std::vector<float>& featureVector,
const std::string& camera_id,
const ImageMetadata& meta);
std::vector<Object> DetectObjects(const cv::Mat& inputImage, const std::string& camera_id);
std::vector<std::vector<Object>> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(const std::vector<cv::Mat>& inputImages,
BatchMetadata& outMetadata);
std::vector<Object> PostProcessPoseBatch(std::vector<float>& featureVector,
const std::string& camera_id,
size_t batchIdx,
const BatchMetadata& metadata);
std::unique_ptr<Engine<float>> m_trtEngine = nullptr;
// Used for image preprocessing
// YoloV8 model expects values between [0.f, 1.f] so we use the following params
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
ANSCENTER::Options m_options;
// Filter thresholds
float PROBABILITY_THRESHOLD;
float NMS_THRESHOLD;
int TOP_K;
// Segmentation constants
int SEG_CHANNELS;
int SEG_H;
int SEG_W;
float SEGMENTATION_THRESHOLD;
// Pose estimation constant
int NUM_KPS;
float KPS_THRESHOLD;
};
}
#endif
File diff suppressed because it is too large Load Diff
+92
View File
@@ -0,0 +1,92 @@
#ifndef ANSTENSORRTSEG_H
#define ANSTENSORRTSEG_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "ANSGpuFrameRegistry.h"
#include "NV12PreprocessHelper.h"
namespace ANSCENTER {
// TensorRT class
class ANSENGINE_API TENSORRTSEG :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
virtual std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs, const std::string& camera_id) override;
bool Destroy();
~TENSORRTSEG();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
// NV12 fast-path helper
NV12PreprocessHelper m_nv12Helper;
private:
// Per-call batch metadata — passed by out-param to avoid shared state
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
// Per-call single-image metadata
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
// Preprocess the input; fills outMeta with per-call image geometry
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage,
ImageMetadata& outMeta);
// Postprocess the output for segmentation model
std::vector<Object> PostProcessSegmentation(std::vector<std::vector<float>>& featureVectors,
const std::string& camera_id,
const ImageMetadata& meta);
std::vector<Object> DetectObjects(const cv::Mat& inputImage, const std::string& camera_id);
std::vector<std::vector<Object>> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(const std::vector<cv::Mat>& inputImages,
BatchMetadata& outMetadata);
std::vector<Object> PostProcessSegmentationBatch(std::vector<std::vector<float>>& featureVectors,
const std::string& camera_id,
size_t batchIdx,
const BatchMetadata& metadata);
std::unique_ptr<Engine<float>> m_trtEngine = nullptr;
std::vector<cv::Point2f> maskToPolygon(const cv::Mat& binaryMask,
const cv::Rect& boundingBox,
float simplificationEpsilon = 2.0f,
int minContourArea = 10);
// Used for image preprocessing
// YoloV8 model expects values between [0.f, 1.f] so we use the following params
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
ANSCENTER::Options m_options;
// Filter thresholds
float PROBABILITY_THRESHOLD;
float NMS_THRESHOLD;
int TOP_K;
// Segmentation constants
int SEG_CHANNELS;
int SEG_H;
int SEG_W;
float SEGMENTATION_THRESHOLD;
// Pose estimation constant
int NUM_KPS;
float KPS_THRESHOLD;
};
}
#endif
File diff suppressed because it is too large Load Diff
+912
View File
@@ -0,0 +1,912 @@
#include "ANSYOLO12OD.h"
#include "EPLoader.h"
#ifdef USEONNXOV
#endif
namespace ANSCENTER {
bool YOLO12OD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
optimizedModelFolder = GetParentFolder(_modelFilePath);
this->_logger.LogDebug("YOLO12OD::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
return true;
}
else {
optimizedModelFolder = "";
this->_logger.LogFatal("YOLO12OD::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
return false;
}
}
bool YOLO12OD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
_modelConfig.modelType = modelType;
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
this->_logger.LogDebug("YOLO12OD::Initialize. Loading YoloV12 weight", _modelFilePath, __FILE__, __LINE__);
if (!FileExist(_modelFilePath)) {
this->_logger.LogError("YOLO12OD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.onnx");
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("YOLO12OD::Initialize. Loading YoloV12 weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("YOLO12OD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
if (FileExist(_classFilePath))
{
this->_logger.LogDebug("YOLO12OD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
else {
this->_logger.LogDebug("YOLO12OD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
}
_isInitialized = loadModel(_modelFilePath, true);//Assume that GPU is available;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("YOLO12OD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool YOLO12OD::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName,std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className,modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string weightFileName = _modelName + ".weights";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
_modelConfig.modelType = modelType;
weightFileName = _modelName + ".onnx";
_modelFilePath = CreateFilePath(_modelFolder, weightFileName);
this->_logger.LogDebug("YOLO12OD::Initialize. Loading YoloV12 weight", _modelFilePath, __FILE__, __LINE__);
if (!FileExist(_modelFilePath)) {
this->_logger.LogError("YOLO12OD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
else {// This is old version of model zip file
weightFileName = _modelName + ".onnx";
std::string onnxfile = CreateFilePath(_modelFolder, weightFileName);
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, className);
this->_logger.LogDebug("YOLO12OD::Initialize. Loading YoloV8 weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("YOLO12OD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("YOLO12OD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("YOLO12OD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
_isInitialized = loadModel(_modelFilePath, true);//Assume that GPU is available;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("YOLO12OD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool YOLO12OD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
_modelConfig.modelType = modelType;
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
this->_logger.LogDebug("YOLO12OD::Initialize. Loading YoloV8 weight", _modelFilePath, __FILE__, __LINE__);
if (!FileExist(_modelFilePath)) {
this->_logger.LogError("YOLO12OD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
}
else {// This is old version of model zip file
std::string onnxfile = CreateFilePath(_modelFolder, "train_last.onnx");
if (std::filesystem::exists(onnxfile)) {
// This is the yovoV5 or yolov8 format
_modelFilePath = onnxfile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("YOLO12OD::Initialize. Loading YoloV8/Yolov5 weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("YOLO12OD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
if (FileExist(_classFilePath))
{
this->_logger.LogDebug("YOLO12OD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
else {
this->_logger.LogDebug("YOLO12OD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
_isInitialized = loadModel(_modelFilePath, true);//Assume that GPU is available;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("YOLO12OD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> YOLO12OD::RunInference(const cv::Mat& input) {
return RunInference(input, "CustomCam");
}
std::vector<Object> YOLO12OD::RunInference(const cv::Mat& input,const std::string& camera_id)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Validation
if (!_licenseValid) {
_logger.LogError("YOLO12OD::RunInference", "Invalid License",
__FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("YOLO12OD::RunInference", "Model is not initialized",
__FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 10 || input.rows < 10) {
return {};
}
auto ret = detect(input, _modelConfig.detectionScoreThreshold,
_modelConfig.modelMNSThreshold);
if (_trackerEnabled) {
ret = ApplyTracking(ret, camera_id);
if (_stabilizationEnabled) ret = StabilizeDetections(ret, camera_id);
}
return ret;
}
catch (const std::exception& e) {
_logger.LogFatal("YOLO12OD::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
YOLO12OD::~YOLO12OD() {
try {
this->_logger.LogDebug("YOLO12OD::~YOLO12OD()", "Release YOLO12OD ", __FILE__, __LINE__);
}
catch (std::exception& e) {
this->_logger.LogFatal("YOLO12OD::~YOLO12OD()", e.what(), __FILE__, __LINE__);
}
}
bool YOLO12OD::Destroy() {
try {
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("YOLO12OD::Destroy()", e.what(), __FILE__, __LINE__);
return false;
}
}
bool YOLO12OD::loadModel(const std::string& modelPath, bool useGPU)
{
try {
const auto& ep = ANSCENTER::EPLoader::Current();
if (Ort::Global<void>::api_ == nullptr)
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
std::cout << "[YOLO12OD] EP ready: "
<< ANSCENTER::EPLoader::EngineTypeName(ep.type) << std::endl;
env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "ONNX_DETECTION");
sessionOptions = Ort::SessionOptions();
sessionOptions.SetIntraOpNumThreads(
std::min(6, static_cast<int>(std::thread::hardware_concurrency())));
sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
// ── Log available providers ─────────────────────────────────────────
std::vector<std::string> availableProviders = Ort::GetAvailableProviders();
std::cout << "Available Execution Providers:" << std::endl;
for (const auto& p : availableProviders)
std::cout << " - " << p << std::endl;
// ── Attach EP based on runtime-detected hardware ────────────────────
if (useGPU) {
bool attached = false;
switch (ep.type) {
case ANSCENTER::EngineType::NVIDIA_GPU: {
auto it = std::find(availableProviders.begin(),
availableProviders.end(), "CUDAExecutionProvider");
if (it == availableProviders.end()) {
this->_logger.LogError("YOLO12OD::loadModel", "CUDAExecutionProvider not in DLL — "
"check ep/cuda/ has the CUDA ORT build.", __FILE__, __LINE__);
break;
}
try {
OrtCUDAProviderOptionsV2* cuda_options = nullptr;
Ort::GetApi().CreateCUDAProviderOptions(&cuda_options);
const char* keys[] = { "device_id" };
const char* values[] = { "0" };
Ort::GetApi().UpdateCUDAProviderOptions(cuda_options, keys, values, 1);
sessionOptions.AppendExecutionProvider_CUDA_V2(*cuda_options);
Ort::GetApi().ReleaseCUDAProviderOptions(cuda_options);
std::cout << "[YOLO12OD] CUDA EP attached." << std::endl;
attached = true;
}
catch (const Ort::Exception& e) {
this->_logger.LogError("YOLO12OD::loadModel", e.what(), __FILE__, __LINE__);
}
break;
}
case ANSCENTER::EngineType::AMD_GPU: {
auto it = std::find(availableProviders.begin(),
availableProviders.end(), "DmlExecutionProvider");
if (it == availableProviders.end()) {
this->_logger.LogError("YOLO12OD::loadModel", "DmlExecutionProvider not in DLL — "
"check ep/directml/ has the DirectML ORT build.", __FILE__, __LINE__);
break;
}
try {
std::unordered_map<std::string, std::string> opts = { { "device_id", "0" } };
sessionOptions.AppendExecutionProvider("DML", opts);
std::cout << "[YOLO12OD] DirectML EP attached." << std::endl;
attached = true;
}
catch (const Ort::Exception& e) {
this->_logger.LogError("YOLO12OD::loadModel", e.what(), __FILE__, __LINE__);
}
break;
}
case ANSCENTER::EngineType::OPENVINO_GPU: {
auto it = std::find(availableProviders.begin(),
availableProviders.end(), "OpenVINOExecutionProvider");
if (it == availableProviders.end()) {
this->_logger.LogError("YOLO12OD::loadModel", "OpenVINOExecutionProvider not in DLL — "
"check ep/openvino/ has the OpenVINO ORT build.", __FILE__, __LINE__);
break;
}
const std::string precision = "FP16";
const std::string numberOfThreads = "8";
const std::string numberOfStreams = "8";
std::vector<std::unordered_map<std::string, std::string>> try_configs = {
{ {"device_type","AUTO:NPU,GPU"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
{ {"device_type","GPU.0"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
{ {"device_type","GPU.1"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} },
{ {"device_type","AUTO:GPU,CPU"}, {"precision",precision},
{"num_of_threads",numberOfThreads}, {"num_streams",numberOfStreams},
{"enable_opencl_throttling","False"}, {"enable_qdq_optimizer","True"} }
};
for (const auto& config : try_configs) {
try {
sessionOptions.AppendExecutionProvider_OpenVINO_V2(config);
std::cout << "[YOLO12OD] OpenVINO EP attached ("
<< config.at("device_type") << ")." << std::endl;
attached = true;
break;
}
catch (const Ort::Exception& e) {
this->_logger.LogError("YOLO12OD::loadModel", e.what(), __FILE__, __LINE__);
}
}
if (!attached)
std::cerr << "[YOLO12OD] OpenVINO EP: all device configs failed." << std::endl;
break;
}
default:
break;
}
if (!attached) {
std::cerr << "[YOLO12OD] No GPU EP attached — running on CPU." << std::endl;
this->_logger.LogFatal("YOLO12OD::loadModel", "GPU EP not attached. Running on CPU.", __FILE__, __LINE__);
}
}
else {
std::cout << "[YOLO12OD] Inference device: CPU (useGPU=false)" << std::endl;
}
// ── Load model ──────────────────────────────────────────────────────
#ifdef _WIN32
std::wstring w_modelPath(modelPath.begin(), modelPath.end());
session = Ort::Session(env, w_modelPath.c_str(), sessionOptions);
#else
session = Ort::Session(env, modelPath.c_str(), sessionOptions);
#endif
Ort::AllocatorWithDefaultOptions allocator;
// ── Input shape ─────────────────────────────────────────────────────
Ort::TypeInfo inputTypeInfo = session.GetInputTypeInfo(0);
std::vector<int64_t> inputTensorShapeVec =
inputTypeInfo.GetTensorTypeAndShapeInfo().GetShape();
isDynamicInputShape = (inputTensorShapeVec.size() >= 4) &&
(inputTensorShapeVec[2] == -1 && inputTensorShapeVec[3] == -1);
if (isDynamicInputShape)
std::cout << "Dynamic input shape detected." << std::endl;
for (auto shape : inputTensorShapeVec)
std::cout << "Input shape: " << shape << std::endl;
// ── Node names ──────────────────────────────────────────────────────
auto input_name = session.GetInputNameAllocated(0, allocator);
inputNodeNameAllocatedStrings.push_back(std::move(input_name));
inputNames.push_back(inputNodeNameAllocatedStrings.back().get());
auto output_name = session.GetOutputNameAllocated(0, allocator);
outputNodeNameAllocatedStrings.push_back(std::move(output_name));
outputNames.push_back(outputNodeNameAllocatedStrings.back().get());
// ── Input image size ────────────────────────────────────────────────
if (inputTensorShapeVec.size() >= 4) {
inputImageShape = cv::Size(
static_cast<int>(inputTensorShapeVec[3]),
static_cast<int>(inputTensorShapeVec[2]));
}
else {
this->_logger.LogFatal("YOLO12OD::loadModel", "Invalid input tensor shape.", __FILE__, __LINE__);
return false;
}
numInputNodes = session.GetInputCount();
numOutputNodes = session.GetOutputCount();
std::cout << "Model loaded successfully with "
<< numInputNodes << " input nodes and "
<< numOutputNodes << " output nodes." << std::endl;
return true;
}
catch (const std::exception& e) {
this->_logger.LogFatal("YOLO12OD::loadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
cv::Mat YOLO12OD::preprocess(const cv::Mat& image, std::vector<float>& blob, std::vector<int64_t>& inputTensorShape) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
m_imgWidth = image.cols;
m_imgHeight = image.rows;
try {
cv::Mat processedImage;
// Handle grayscale images by converting them to 3-channel BGR
if (image.channels() == 1) {
cv::cvtColor(image, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = image.clone();
}
cv::Mat resizedImage;
// Resize and pad the image using letterBox utility
letterBox(processedImage, resizedImage, inputImageShape, cv::Scalar(114, 114, 114), isDynamicInputShape, false, true, 32);
// Update input tensor shape based on resized image dimensions
inputTensorShape[2] = resizedImage.rows;
inputTensorShape[3] = resizedImage.cols;
// Convert image to float and normalize to [0, 1]
resizedImage.convertTo(resizedImage, CV_32FC3, 1 / 255.0f);
// Allocate memory for the image blob in CHW format
size_t totalSize = resizedImage.cols * resizedImage.rows * resizedImage.channels();
blob.resize(totalSize);
// Split the image into separate channels and store in the blob
std::vector<cv::Mat> chw(resizedImage.channels());
for (int i = 0; i < resizedImage.channels(); ++i) {
chw[i] = cv::Mat(resizedImage.rows, resizedImage.cols, CV_32FC1, blob.data() + i * resizedImage.cols * resizedImage.rows);
}
cv::split(resizedImage, chw); // Split channels into the blob
return resizedImage;
}
catch (const std::exception& e) {
this->_logger.LogFatal("YOLO12OD::preprocess", e.what(), __FILE__, __LINE__);
return cv::Mat();
}
}
std::vector<Object> YOLO12OD::postprocess(const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
float confThreshold, float iouThreshold)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
std::vector<Object> detections;
const float* rawOutput = outputTensors[0].GetTensorData<float>(); // Extract raw output data from the first output tensor
const std::vector<int64_t> outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
// Determine the number of features and detections
const size_t num_features = outputShape[1];
const size_t num_detections = outputShape[2];
// Early exit if no detections
if (num_detections == 0) {
return detections;
}
// Calculate number of classes based on output shape
const int numClasses = static_cast<int>(num_features) - 4;
if (numClasses <= 0) {
// Invalid number of classes
return detections;
}
// Reserve memory for efficient appending
std::vector<BoundingBox> boxes;
boxes.reserve(num_detections);
std::vector<float> confs;
confs.reserve(num_detections);
std::vector<int> classIds;
classIds.reserve(num_detections);
std::vector<BoundingBox> nms_boxes;
nms_boxes.reserve(num_detections);
// Constants for indexing
const float* ptr = rawOutput;
for (size_t d = 0; d < num_detections; ++d) {
// Extract bounding box coordinates (center x, center y, width, height)
float centerX = ptr[0 * num_detections + d];
float centerY = ptr[1 * num_detections + d];
float width = ptr[2 * num_detections + d];
float height = ptr[3 * num_detections + d];
// Find class with the highest confidence score
int classId = -1;
float maxScore = -FLT_MAX;
for (int c = 0; c < numClasses; ++c) {
const float score = ptr[d + (4 + c) * num_detections];
if (score > maxScore) {
maxScore = score;
classId = c;
}
}
// Proceed only if confidence exceeds threshold
if (maxScore > confThreshold) {
// Convert center coordinates to top-left (x1, y1)
float left = centerX - width / 2.0f;
float top = centerY - height / 2.0f;
// Scale to original image size
BoundingBox scaledBox = scaleCoords(
resizedImageShape,
BoundingBox(left, top, width, height),
originalImageSize,
true
);
// Round coordinates for integer pixel positions
BoundingBox roundedBox;
roundedBox.x = std::round(scaledBox.x);
roundedBox.y = std::round(scaledBox.y);
roundedBox.width = std::round(scaledBox.width);
roundedBox.height = std::round(scaledBox.height);
// Adjust NMS box coordinates to prevent overlap between classes
BoundingBox nmsBox = roundedBox;
nmsBox.x += classId * 7680; // Arbitrary offset to differentiate classes
nmsBox.y += classId * 7680;
// Add to respective containers
nms_boxes.emplace_back(nmsBox);
boxes.emplace_back(roundedBox);
confs.emplace_back(maxScore);
classIds.emplace_back(classId);
}
}
// Apply Non-Maximum Suppression (NMS) to eliminate redundant detections
std::vector<int> indices;
NMSBoxes(nms_boxes, confs, confThreshold, iouThreshold, indices);
// Collect filtered detections into the result vector
int classNameSize = _classes.size();
detections.reserve(indices.size());
for (const int idx : indices) {
float conf = confs[idx];
if (conf >= confThreshold) {
Object detection;
detection.confidence = confs[idx];
detection.box.x = boxes[idx].x;
detection.box.y = boxes[idx].y;
detection.box.width = boxes[idx].width;
detection.box.height = boxes[idx].height;
detection.classId = classIds[idx];
if (!_classes.empty()) {
if (detection.classId < classNameSize) {
detection.className = _classes[detection.classId];
}
else {
detection.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
}
}
else {
detection.className = "Unknown"; // Fallback if _classes is empty
}
detection.polygon = ANSUtilityHelper::RectToNormalizedPolygon(detection.box, m_imgWidth, m_imgHeight);
detections.push_back(detection);
}
}
return detections;
}
catch (const std::exception& e) {
this->_logger.LogFatal("YOLO12OD::postprocess", e.what(), __FILE__, __LINE__);
return std::vector<Object>();
}
}
std::vector<Object> YOLO12OD::detect(const cv::Mat& image, float confThreshold, float iouThreshold) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
// Define the shape of the input tensor (batch size, channels, height, width)
std::vector<int64_t> inputTensorShape = { 1, 3, inputImageShape.height, inputImageShape.width };
// Preprocess the image and obtain the blob as a vector<float>
std::vector<float> blob;
cv::Mat preprocessedImage = preprocess(image, blob, inputTensorShape);
// Compute the total number of elements in the input tensor
size_t inputTensorSize = vectorProduct(inputTensorShape);
if (blob.size() != inputTensorSize) {
this->_logger.LogFatal("YOLO12OD::detect", "Mismatch between blob size and expected tensor size", __FILE__, __LINE__);
return {};
}
// Create an Ort memory info object (cached for efficiency)
static const Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
// Create input tensor object using the preprocessed data
Ort::Value inputTensor = Ort::Value::CreateTensor<float>(
memoryInfo,
blob.data(), // Use the vector's data directly
inputTensorSize,
inputTensorShape.data(),
inputTensorShape.size()
);
// Run the inference session with the input tensor and retrieve output tensors
std::vector<Ort::Value> outputTensors = session.Run(
Ort::RunOptions{ nullptr },
inputNames.data(),
&inputTensor,
numInputNodes,
outputNames.data(),
numOutputNodes
);
// Determine the resized image shape based on input tensor shape
cv::Size resizedImageShape(static_cast<int>(inputTensorShape[3]), static_cast<int>(inputTensorShape[2]));
// Postprocess the output tensors to obtain detections
std::vector<Object> detections = postprocess(image.size(), resizedImageShape, outputTensors, confThreshold, iouThreshold);
return detections; // Return the vector of detections
}
catch (const std::exception& e) {
this->_logger.LogFatal("YOLO12OD::detect", e.what(), __FILE__, __LINE__);
return {};
}
}
// Utility function to clamp a value within a specified range
size_t YOLO12OD::vectorProduct(const std::vector<int64_t>& vector) {
return std::accumulate(vector.begin(), vector.end(), 1ull, std::multiplies<size_t>());
}
void YOLO12OD::letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color,
bool auto_,
bool scaleFill,
bool scaleUp,
int stride)
{
try {
// Calculate the scaling ratio to fit the image within the new shape
float ratio = std::min(static_cast<float>(newShape.height) / image.rows,
static_cast<float>(newShape.width) / image.cols);
// Prevent scaling up if not allowed
if (!scaleUp) {
ratio = std::min(ratio, 1.0f);
}
// Calculate new dimensions after scaling
int newUnpadW = static_cast<int>(std::round(image.cols * ratio));
int newUnpadH = static_cast<int>(std::round(image.rows * ratio));
// Calculate padding needed to reach the desired shape
int dw = newShape.width - newUnpadW;
int dh = newShape.height - newUnpadH;
if (auto_) {
// Ensure padding is a multiple of stride for model compatibility
dw = (dw % stride) / 2;
dh = (dh % stride) / 2;
}
else if (scaleFill) {
// Scale to fill without maintaining aspect ratio
newUnpadW = newShape.width;
newUnpadH = newShape.height;
ratio = std::min(static_cast<float>(newShape.width) / image.cols,
static_cast<float>(newShape.height) / image.rows);
dw = 0;
dh = 0;
}
else {
// Evenly distribute padding on both sides
// Calculate separate padding for left/right and top/bottom to handle odd padding
int padLeft = dw / 2;
int padRight = dw - padLeft;
int padTop = dh / 2;
int padBottom = dh - padTop;
// Resize the image if the new dimensions differ
if (image.cols != newUnpadW || image.rows != newUnpadH) {
cv::resize(image, outImage, cv::Size(newUnpadW, newUnpadH), 0, 0, cv::INTER_LINEAR);
}
else {
// Avoid unnecessary copying if dimensions are the same
outImage = image;
}
// Apply padding to reach the desired shape
cv::copyMakeBorder(outImage, outImage, padTop, padBottom, padLeft, padRight, cv::BORDER_CONSTANT, color);
return; // Exit early since padding is already applied
}
// Resize the image if the new dimensions differ
if (image.cols != newUnpadW || image.rows != newUnpadH) {
cv::resize(image, outImage, cv::Size(newUnpadW, newUnpadH), 0, 0, cv::INTER_LINEAR);
}
else {
// Avoid unnecessary copying if dimensions are the same
outImage = image;
}
// Calculate separate padding for left/right and top/bottom to handle odd padding
int padLeft = dw / 2;
int padRight = dw - padLeft;
int padTop = dh / 2;
int padBottom = dh - padTop;
// Apply padding to reach the desired shape
cv::copyMakeBorder(outImage, outImage, padTop, padBottom, padLeft, padRight, cv::BORDER_CONSTANT, color);
}
catch (const std::exception& e) {
std::cerr << "Error in letterBox: " << e.what() << std::endl;
}
}
BoundingBox YOLO12OD::scaleCoords(const cv::Size& imageShape, BoundingBox coords,
const cv::Size& imageOriginalShape, bool p_Clip)
{
BoundingBox result;
try {
float gain = std::min(static_cast<float>(imageShape.height) / static_cast<float>(imageOriginalShape.height),
static_cast<float>(imageShape.width) / static_cast<float>(imageOriginalShape.width));
int padX = static_cast<int>(std::round((imageShape.width - imageOriginalShape.width * gain) / 2.0f));
int padY = static_cast<int>(std::round((imageShape.height - imageOriginalShape.height * gain) / 2.0f));
result.x = static_cast<int>(std::round((coords.x - padX) / gain));
result.y = static_cast<int>(std::round((coords.y - padY) / gain));
result.width = static_cast<int>(std::round(coords.width / gain));
result.height = static_cast<int>(std::round(coords.height / gain));
if (p_Clip) {
result.x = clamp(result.x, 0, imageOriginalShape.width);
result.y = clamp(result.y, 0, imageOriginalShape.height);
result.width = clamp(result.width, 0, imageOriginalShape.width - result.x);
result.height = clamp(result.height, 0, imageOriginalShape.height - result.y);
}
return result;
}
catch (const std::exception& e) {
std::cerr << "Error in scaleCoords: " << e.what() << std::endl;
return result;
}
}
// Optimized Non-Maximum Suppression Function
void YOLO12OD::NMSBoxes(const std::vector<BoundingBox>& boundingBoxes,
const std::vector<float>& scores,
float scoreThreshold,
float nmsThreshold,
std::vector<int>& indices)
{
indices.clear();
const size_t numBoxes = boundingBoxes.size();
if (numBoxes == 0) {
return;
}
// Step 1: Filter out boxes with scores below the threshold
// and create a list of indices sorted by descending scores
std::vector<int> sortedIndices;
sortedIndices.reserve(numBoxes);
for (size_t i = 0; i < numBoxes; ++i) {
if (scores[i] >= scoreThreshold) {
sortedIndices.push_back(static_cast<int>(i));
}
}
// If no boxes remain after thresholding
if (sortedIndices.empty()) {
return;
}
// Sort the indices based on scores in descending order
std::sort(sortedIndices.begin(), sortedIndices.end(),
[&scores](int idx1, int idx2) {
return scores[idx1] > scores[idx2];
});
// Step 2: Precompute the areas of all boxes
std::vector<float> areas(numBoxes, 0.0f);
for (size_t i = 0; i < numBoxes; ++i) {
areas[i] = boundingBoxes[i].width * boundingBoxes[i].height;
}
// Step 3: Suppression mask to mark boxes that are suppressed
std::vector<bool> suppressed(numBoxes, false);
// Step 4: Iterate through the sorted list and suppress boxes with high IoU
for (size_t i = 0; i < sortedIndices.size(); ++i) {
int currentIdx = sortedIndices[i];
if (suppressed[currentIdx]) {
continue;
}
// Select the current box as a valid detection
indices.push_back(currentIdx);
const BoundingBox& currentBox = boundingBoxes[currentIdx];
const float x1_max = currentBox.x;
const float y1_max = currentBox.y;
const float x2_max = currentBox.x + currentBox.width;
const float y2_max = currentBox.y + currentBox.height;
const float area_current = areas[currentIdx];
// Compare IoU of the current box with the rest
for (size_t j = i + 1; j < sortedIndices.size(); ++j) {
int compareIdx = sortedIndices[j];
if (suppressed[compareIdx]) {
continue;
}
const BoundingBox& compareBox = boundingBoxes[compareIdx];
const float x1 = std::max(x1_max, static_cast<float>(compareBox.x));
const float y1 = std::max(y1_max, static_cast<float>(compareBox.y));
const float x2 = std::min(x2_max, static_cast<float>(compareBox.x + compareBox.width));
const float y2 = std::min(y2_max, static_cast<float>(compareBox.y + compareBox.height));
const float interWidth = x2 - x1;
const float interHeight = y2 - y1;
if (interWidth <= 0 || interHeight <= 0) {
continue;
}
const float intersection = interWidth * interHeight;
const float unionArea = area_current + areas[compareIdx] - intersection;
const float iou = (unionArea > 0.0f) ? (intersection / unionArea) : 0.0f;
if (iou > nmsThreshold) {
suppressed[compareIdx] = true;
}
}
}
}
}
+89
View File
@@ -0,0 +1,89 @@
#ifndef ANSYOLO12OD_H
#define ANSYOL12OOD_H
#pragma once
#include "ANSEngineCommon.h"
#include <onnxruntime_cxx_api.h>
#include <algorithm>
#include <fstream>
#include <iostream>
#include <numeric>
#include <string>
#include <vector>
#include <memory>
#include <chrono>
#include <random>
#include <unordered_map>
#include <thread>
namespace ANSCENTER {
// Standard Yolo engine class: yolo12
class ANSENGINE_API YOLO12OD :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~YOLO12OD();
private:
Ort::Env env{ nullptr }; // ONNX Runtime environment
Ort::SessionOptions sessionOptions{ nullptr }; // Session options for ONNX Runtime
Ort::Session session{ nullptr }; // ONNX Runtime session for running inference
bool isDynamicInputShape{}; // Flag indicating if input shape is dynamic
cv::Size inputImageShape; // Expected input image shape for the model
std::string _modelFilePath;
// Vectors to hold allocated input and output node names
std::vector<Ort::AllocatedStringPtr> inputNodeNameAllocatedStrings;
std::vector<const char*> inputNames;
std::vector<Ort::AllocatedStringPtr> outputNodeNameAllocatedStrings;
std::vector<const char*> outputNames;
size_t numInputNodes, numOutputNodes; // Number of input and output nodes in the model
std::vector<std::string> classNames; // Vector of class names loaded from file
std::vector<cv::Scalar> classColors; // Vector of colors for each class
float m_imgWidth = 0;
float m_imgHeight = 0;
protected:
bool loadModel(const std::string& modelPath, bool useGPU = true);
std::vector<Object> detect(const cv::Mat& image, float confThreshold = 0.4f, float iouThreshold = 0.45f);
//cv::Mat preprocess(const cv::Mat& image, float*& blob, std::vector<int64_t>& inputTensorShape);
cv::Mat preprocess(const cv::Mat& image, std::vector<float>& blob, std::vector<int64_t>& inputTensorShape);
std::vector<Object> postprocess(const cv::Size& originalImageSize, const cv::Size& resizedImageShape,
const std::vector<Ort::Value>& outputTensors,
float confThreshold, float iouThreshold);
private:
template <typename T>
typename std::enable_if<std::is_arithmetic<T>::value, T>::type
inline clamp(const T& value, const T& low, const T& high)
{
// Ensure the range [low, high] is valid; swap if necessary
T validLow = low < high ? low : high;
T validHigh = low < high ? high : low;
// Clamp the value to the range [validLow, validHigh]
if (value < validLow)
return validLow;
if (value > validHigh)
return validHigh;
return value;
}
size_t vectorProduct(const std::vector<int64_t>& vector);
void letterBox(const cv::Mat& image, cv::Mat& outImage,
const cv::Size& newShape,
const cv::Scalar& color = cv::Scalar(114, 114, 114),
bool auto_ = true,
bool scaleFill = false,
bool scaleUp = true,
int stride = 32);
BoundingBox scaleCoords(const cv::Size& imageShape, BoundingBox coords,
const cv::Size& imageOriginalShape, bool p_Clip);
void NMSBoxes(const std::vector<BoundingBox>& boundingBoxes,
const std::vector<float>& scores,
float scoreThreshold,
float nmsThreshold,
std::vector<int>& indices);
};
}
#endif
File diff suppressed because it is too large Load Diff
+64
View File
@@ -0,0 +1,64 @@
#ifndef ANSYOLOOD_H
#define ANSYOLOOD_H
#pragma once
#include "ANSEngineCommon.h"
#include <onnxruntime_cxx_api.h>
//#define USEOPENCVDNN
namespace ANSCENTER {
class ANSENGINE_API YOLOOD :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~YOLOOD();
private:
std::shared_ptr<cv::dnn::Net> _net;
std::string _modelFilePath;
bool _letterBoxForSquare{ true };
bool _isDarkNet{false};
std::vector<float> _blob; // Reusable buffer
cv::Mat _frameBuffer; // Reusable buffer for color conversion
private:
std::vector<Object> RunInferenceFromONNX(const cv::Mat& input, const std::string& camera_id);
std::vector<Object> RunInferenceFromYoloV5(const cv::Mat& input, const std::string& camera_id);
std::vector<Object> RunInferenceFromYoloV11(const cv::Mat& input, const std::string& camera_id);
std::vector<Object> RunInferenceFromDarkNet(const cv::Mat& input, const std::string& camera_id);
std::vector<Object> RunInferenceFromYoloV8(const cv::Mat& input, const std::string& camera_id);// OpenCV Style
void LoadOnnxNetwork();
void LoadDarknetNetwork();
void SetEngineType();
private:
Ort::Env env{ nullptr };
Ort::SessionOptions sessionOptions{ nullptr };
Ort::Session session{ nullptr };
bool loadYoloModel(const std::string& modelPath,const bool& isGPU = true,const cv::Size& inputSize = cv::Size(640, 640));
std::vector<Object> detect(cv::Mat& image, const float& confThreshold, const float& iouThreshold);
void preprocessing(const cv::Mat& image,std::vector<float>& blob,std::vector<int64_t>& inputTensorShape);
//cv::Mat preprocessv11(const cv::Mat& image, float*& blob, std::vector<int64_t>& inputTensorShape);
cv::Mat preprocessv11(const cv::Mat& image, std::vector<float>& blob, std::vector<int64_t>& inputTensorShape);
std::vector<Object> postprocessing(const cv::Size& resizedImageShape,const cv::Size& originalImageShape,std::vector<Ort::Value>& outputTensors,
const float& confThreshold, const float& iouThreshold);
std::vector<Object> postprocessv11(const cv::Size& originalImageSize,const cv::Size& resizedImageShape,const std::vector<Ort::Value>& outputTensors,float confThreshold,float iouThreshold);
BoundingBox scaleCoordsv11(const cv::Size& imageShape, BoundingBox coords,const cv::Size& imageOriginalShape, bool p_Clip);
std::vector<const char*> inputNodeNames;
std::vector<const char*> outputNodeNames;
bool isDynamicInputShape{};
cv::Size inputImageShape; // Expected input image shape for the model
float resizeScales;//letterbox scale
float m_imgWidth = 0;
float m_imgHeight = 0;
std::vector<Object> detectv11(const cv::Mat& image, float confThreshold, float iouThreshold);
//ultility functions
void getBestClassInfo(std::vector<float>::iterator it, const int& numClasses,float& bestConf, int& bestClassId);
void letterbox(const cv::Mat& image, cv::Mat& outImage,const cv::Size& newShape,const cv::Scalar& color,bool auto_,bool scaleFill,bool scaleUp,int stride);
void scaleCoords(const cv::Size& imageShape, cv::Rect& box, const cv::Size& imageOriginalShape);
size_t vectorProduct(const std::vector<int64_t>& vector);
};
}
#endif
+977
View File
@@ -0,0 +1,977 @@
#include "ANSYOLOV10OVOD.h"
#include "Utility.h"
namespace ANSCENTER
{
bool ANSOYOLOV10OVOD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (FileExist(_modelFilePath)) {
std::string modelName = GetFileNameWithoutExtension(_modelFilePath);
std::string binaryModelName = modelName + ".bin";
std::string modelFolder = GetParentFolder(_modelFilePath);
std::string optimizedModelPath = CreateFilePath(modelFolder, binaryModelName);
if (FileExist(optimizedModelPath)) {
this->_logger.LogDebug("ANSOYOLOV10OVOD::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return true;
}
else {
this->_logger.LogFatal("ANSOYOLOV10OVOD::OptimizeModel", "This model can not be optimized.", __FILE__, __LINE__);
optimizedModelFolder = modelFolder;
return false;
}
}
else {
this->_logger.LogFatal("ANSOYOLOV10OVOD::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
optimizedModelFolder = "";
return false;
}
}
bool ANSOYOLOV10OVOD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlModelFile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(xmlModelFile)) {
_modelFilePath = xmlModelFile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOYOLOV10OVOD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Load Model from Here
InitialModel(_modelFilePath);
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOYOLOV10OVOD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOYOLOV10OVOD::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName,std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig,modelName, className,modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".xml";
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlModelFile = CreateFilePath(_modelFolder, modelFullName);//yolov8n.xml
if (std::filesystem::exists(xmlModelFile)) {
_modelFilePath = xmlModelFile;
_classFilePath = CreateFilePath(_modelFolder, className);
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOYOLOV10OVOD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel(_modelFilePath);
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOYOLOV10OVOD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOYOLOV10OVOD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
std::string openVINOVersion = ov::get_openvino_version().buildNumber;
std::cout << "OpenVINO version: " << openVINOVersion << std::endl;
//this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. OpenVINO version", openVINOVersion, __FILE__, __LINE__);
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::OPENVINO;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
std::string xmlModelFile = CreateFilePath(_modelFolder, "train_last.xml");//yolov8n.xml
if (std::filesystem::exists(xmlModelFile)) {
_modelFilePath = xmlModelFile;
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Loading OpenVINO weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOYOLOV10OVOD::Initialize. Model file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSOYOLOV10OVOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load Model from Here
InitialModel(_modelFilePath);
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOYOLOV10OVOD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSOYOLOV10OVOD::RunInference(const cv::Mat& input) {
return RunInference(input, "OpenVINO10Cam");
}
std::vector<Object> ANSOYOLOV10OVOD::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
// Validate license and initialization status
if (!_licenseValid) {
_logger.LogError("ANSOYOLOV10OVOD::RunInference", "Invalid License", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("ANSOYOLOV10OVOD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return {};
}
// Validate input
if (input.empty()) {
_logger.LogError("ANSOYOLOV10OVOD::RunInference", "Input frame is empty", __FILE__, __LINE__);
return {};
}
if (input.cols < 10 || input.rows < 10) {
return {};
}
try {
m_imgWidth = static_cast<float>(input.cols);
m_imgHeight = static_cast<float>(input.rows);
constexpr int imageSize = 640;
float factor = 0.0f;
// Shallow copy - Preprocessing may modify headers but not original data
this->_frameBuffer = input;
// Reuse member buffer for input data
this->_inputData.resize(imageSize * imageSize * 3);
// Preprocess the input frame
Preprocessing(&this->_frameBuffer, imageSize, &factor, this->_inputData);
// Ensure request is valid
if (!request) {
request = compiled_model.create_infer_request();
request.get_input_tensor().set_shape({ 1, 3,
static_cast<unsigned long>(imageSize),
static_cast<unsigned long>(imageSize) });
}
// Get and validate input tensor
auto inputTensor = request.get_input_tensor();
float* tensorData = inputTensor.data<float>();
if (!tensorData) {
_logger.LogError("ANSOYOLOV10OVOD::RunInference", "Input tensor data pointer is null", __FILE__, __LINE__);
return {};
}
// Copy input data into the tensor
std::memcpy(tensorData, this->_inputData.data(), this->_inputData.size() * sizeof(float));
// Run inference
request.infer();
// Retrieve and validate output tensor
auto outputTensor = request.get_output_tensor();
float* output_data = outputTensor.data<float>();
if (!output_data) {
_logger.LogError("ANSOYOLOV10OVOD::RunInference", "Output tensor data pointer is null", __FILE__, __LINE__);
return {};
}
// Post-process and return
auto ret = PostProcessing(output_data, factor, 300, camera_id);
if (_trackerEnabled) {
ret = ApplyTracking(ret, camera_id);
if (_stabilizationEnabled) ret = StabilizeDetections(ret, camera_id);
}
return ret;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOYOLOV10OVOD::RunInference", e.what(), __FILE__, __LINE__);
}
catch (...) {
_logger.LogFatal("ANSOYOLOV10OVOD::RunInference", "An unknown error occurred", __FILE__, __LINE__);
}
return {};
}
cv::Mat ANSOYOLOV10OVOD::resizeKeepAspectRatioPadRightBottom(const cv::Mat& input,size_t height, size_t width,const cv::Scalar& bgcolor) {
// Ensure input is valid
if (input.empty()) {
return cv::Mat();
}
// Calculate aspect ratio and unpadded dimensions
float r = std::min(static_cast<float>(width) / input.cols, static_cast<float>(height) / input.rows);
int unpad_w = static_cast<int>(r * input.cols);
int unpad_h = static_cast<int>(r * input.rows);
// Resize the input image
cv::Mat resized;
cv::resize(input, resized, cv::Size(unpad_w, unpad_h), 0, 0, cv::INTER_CUBIC);
// Create the output image and fill with the background color
cv::Mat output(height, width, input.type(), bgcolor);
// Copy the resized content into the top-left corner of the output image
resized.copyTo(output(cv::Rect(0, 0, resized.cols, resized.rows)));
return output;
}
ANSOYOLOV10OVOD::~ANSOYOLOV10OVOD() {
try {
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogDebug("ANSOYOLOV10OVOD::~ANSOYOLOV10OVOD", "Failed to delete OpenVINO Models", __FILE__, __LINE__);
}
}
}
catch (std::exception& e) {
this->_logger.LogError("ANSOYOLOV10OVOD::~ANSOYOLOV10OVOD()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
}
}
bool ANSOYOLOV10OVOD::Destroy() {
try {
if (FolderExist(_modelFolder)) {
if (!DeleteFolder(_modelFolder)) {
this->_logger.LogDebug("ANSOYOLOV10OVOD::Destroy", "Failed to delete OpenVINO Models", __FILE__, __LINE__);
}
}
return true;
}
catch (std::exception& e) {
this->_logger.LogError("ANSOYOLOV10OVOD::Destroy()", "Failed to release OPENVINO Models", __FILE__, __LINE__);
return false;
}
}
//private
void ANSOYOLOV10OVOD::InitialModel(const std::string& model_path)
{
if((_modelConfig.gpuOptBatchSize==1)&&(_modelConfig.gpuMaxBatchSize==1)) {
InitModelStaticBatchSize(model_path);
}
else {// Dynamic batch size is not supported in OpenVINO for Yolov10
try {
// Step 1: Initialize OpenVINO Runtime Core
int maxBatchSize = _modelConfig.gpuMaxBatchSize;
ov::Core core;
auto model = core.read_model(model_path);
// Step 2: Configure model for dynamic batch size
_logger.LogDebug("ANSOYOLOV10OVOD::InitialModel",
"Configuring model for dynamic batch (max: " + std::to_string(maxBatchSize) + ")",
__FILE__, __LINE__);
// Get input info
auto input = model->get_parameters()[0];
auto input_shape = input->get_partial_shape();
// Set batch dimension to dynamic range [1, maxBatchSize]
input_shape[0] = ov::Dimension(1, maxBatchSize);
// Reshape model for dynamic batch processing
model->reshape({ {input->get_friendly_name(), input_shape} });
// Step 3: Get Available Devices and Log
std::vector<std::string> available_devices = core.get_available_devices();
_logger.LogDebug("ANSOYOLOV10OVOD::InitialModel",
"Available devices: " + std::to_string(available_devices.size()),
__FILE__, __LINE__);
for (const auto& device : available_devices) {
_logger.LogDebug("ANSOYOLOV10OVOD::InitialModel", " - " + device, __FILE__, __LINE__);
}
// Define device priority: GPU > CPU
std::vector<std::string> priority_devices = { "GPU", "CPU" };
bool device_found = false;
std::string selected_device;
// Iterate over prioritized devices
for (const auto& device : priority_devices) {
if (std::find(available_devices.begin(), available_devices.end(), device) != available_devices.end()) {
selected_device = device;
// Configure device-specific properties
ov::AnyMap config;
if (device == "GPU") {
// GPU-specific optimizations for batch processing
config[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::THROUGHPUT;
config[ov::hint::inference_precision.name()] = ov::element::f16;
config["NUM_STREAMS"] = "1"; // Single stream for batching
}
else if (device == "CPU") {
// CPU-specific optimizations for batch processing
config[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::THROUGHPUT;
config["NUM_STREAMS"] = "1"; // Single stream for batching
}
compiled_model = core.compile_model(model, device, config);
device_found = true;
_logger.LogDebug("ANSOYOLOV10OVOD::InitialModel",
"Model compiled on device: " + device, __FILE__, __LINE__);
break;
}
}
// Fallback: Default to CPU if no devices found
if (!device_found) {
_logger.LogError("ANSOYOLOV10OVOD::InitialModel",
"No supported devices found. Falling back to CPU.", __FILE__, __LINE__);
ov::AnyMap cpu_config;
cpu_config[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::THROUGHPUT;
compiled_model = core.compile_model(model, "CPU", cpu_config);
selected_device = "CPU";
}
// Step 4: Create Inference Request
request = compiled_model.create_infer_request();
// Set initial shape to batch size 1
request.get_input_tensor().set_shape({ 1, 3, 640, 640 });
_logger.LogDebug("ANSOYOLOV10OVOD::InitialModel",
"Model initialization complete on " + selected_device +
" with dynamic batch support (1-" + std::to_string(maxBatchSize) + ")",
__FILE__, __LINE__);
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOYOLOV10OVOD::InitialModel", e.what(), __FILE__, __LINE__);
}
}
}
void ANSOYOLOV10OVOD::InitModelStaticBatchSize(const std::string& model_path) {
try {
// Step 1: Initialize OpenVINO Runtime Core
ov::Core core;
auto model = core.read_model(model_path);
// Step 2: Get Available Devices and Log
std::vector<std::string> available_devices = core.get_available_devices();
// Yolov10 does not support NPU, so we remove it from the list
// Define device priority: GPU > CPU
std::vector<std::string> priority_devices = { "GPU","CPU" };
bool device_found = false;
// Iterate over prioritized devices
for (const auto& device : priority_devices) {
if (std::find(available_devices.begin(), available_devices.end(), device) != available_devices.end()) {
compiled_model = core.compile_model(_modelFilePath, device);
device_found = true;
break;
}
}
// Fallback: Default to CPU if no devices found
if (!device_found) {
_logger.LogError("OPENVINOOD::InitialModel", "No supported devices found. Falling back to CPU.", __FILE__, __LINE__);
compiled_model = core.compile_model(model, "CPU");
}
// Step 3: Create Inference Request
request = compiled_model.create_infer_request();
request.get_input_tensor().set_shape({ 1, 3, 640, 640 });
}
catch (const std::exception& e) {
// Log any errors
this->_logger.LogFatal("OPENVINOOD::InitialModel", e.what(), __FILE__, __LINE__);
}
}
void ANSOYOLOV10OVOD::Preprocessing(cv::Mat* img, int length, float* factor, std::vector<float>& data) {
try {
if (!img || img->empty()) {
this->_logger.LogFatal("ANSOYOLOV10OVOD::Preprocessing", "Invalid or empty image", __FILE__, __LINE__);
return;
}
// Convert grayscale images to 3-channel BGR
cv::Mat processedImage;
if (img->channels() == 1) {
cv::cvtColor(*img, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = *img;
}
// Convert image to RGB
cv::Mat rgbImage;
cv::cvtColor(processedImage, rgbImage, cv::COLOR_BGR2RGB);
// Determine the maximum image length
int max_image_length = std::max(rgbImage.cols, rgbImage.rows);
// Create a square image with the maximum length and fill it with white
cv::Mat max_image = cv::Mat::ones(max_image_length, max_image_length, CV_8UC3) * 255;
// Copy the original image to the top-left corner of the square image
cv::Rect roi(0, 0, rgbImage.cols, rgbImage.rows);
rgbImage.copyTo(max_image(roi));
// Resize the image to the desired length
cv::Mat resizedImage;
cv::resize(max_image, resizedImage, cv::Size(length, length), 0.0f, 0.0f, cv::INTER_CUBIC);
// Calculate the scaling factor
*factor = static_cast<float>(max_image_length) / static_cast<float>(length);
// Convert the image to float and normalize
resizedImage.convertTo(resizedImage, CV_32FC3, 1.0 / 255.0);
// Pre-allocate the data vector
data.resize(length * length * 3);
// Split channels and copy data
std::vector<cv::Mat> channels(3);
cv::split(resizedImage, channels);
for (int i = 0; i < 3; ++i) {
std::memcpy(data.data() + i * length * length, channels[i].data, length * length * sizeof(float));
}
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSOYOLOV10OVOD::Preprocessing", e.what(), __FILE__, __LINE__);
}
}
std::vector<Object> ANSOYOLOV10OVOD::PostProcessing(float* result, float factor, int outputLength, const std::string& camera_id) {
std::vector<Object> objects;
try {
std::vector<cv::Rect> position_boxes;
std::vector<int> class_ids;
std::vector<float> confidences;
// Reserve memory to avoid multiple reallocations
position_boxes.reserve(outputLength);
class_ids.reserve(outputLength);
confidences.reserve(outputLength);
for (int i = 0; i < outputLength; ++i) {
int s = 6 * i;
float confidence = result[s + 4];
if (confidence > _modelConfig.detectionScoreThreshold) {
float cx = result[s + 0];
float cy = result[s + 1];
float dx = result[s + 2];
float dy = result[s + 3];
int x = static_cast<int>(cx * factor);
int y = static_cast<int>(cy * factor);
int width = static_cast<int>((dx - cx) * factor);
int height = static_cast<int>((dy - cy) * factor);
x = std::max(0, x);
y = std::max(0, y);
width = std::min(static_cast<int>(m_imgWidth) - x, width);
height = std::min(static_cast<int>(m_imgHeight) - y, height);
cv::Rect box(x, y, width, height);
position_boxes.emplace_back(box);
class_ids.emplace_back(static_cast<int>(result[s + 5]));
confidences.emplace_back(confidence);
}
}
int classNameSize = _classes.size();
objects.reserve(position_boxes.size()); // Reserve memory for objects
for (size_t i = 0; i < position_boxes.size(); ++i) {
Object obj;
obj.classId = class_ids[i];
if (!_classes.empty()) {
if (obj.classId < classNameSize) {
obj.className = _classes[obj.classId];
}
else {
obj.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
}
}
else {
obj.className = "Unknown"; // Fallback if _classes is empty
}
obj.confidence = confidences[i];
obj.box = position_boxes[i];
obj.polygon = ANSUtilityHelper::RectToNormalizedPolygon(obj.box, m_imgWidth, m_imgHeight);
obj.cameraId = camera_id;
objects.emplace_back(std::move(obj));
}
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSOYOLOV10OVOD::PostProcessing", e.what(), __FILE__, __LINE__);
}
//EnqueueDetection(objects,camera_id);
return objects;
}
void ANSOYOLOV10OVOD::PreprocessingBatch(const std::vector<cv::Mat>& images,
int length,
std::vector<float>& factors,
std::vector<float>& data) {
try {
if (images.empty()) {
_logger.LogFatal("ANSOYOLOV10OVOD::PreprocessingBatch", "Empty images vector", __FILE__, __LINE__);
return;
}
int batchSize = static_cast<int>(images.size());
// Store original image dimensions for each image in batch
m_batchImgHeights.resize(batchSize);
m_batchImgWidths.resize(batchSize);
factors.resize(batchSize);
// Pre-allocate data vector for entire batch: [batch, channels, height, width]
data.resize(batchSize * 3 * length * length);
// Process each image in the batch
for (int b = 0; b < batchSize; ++b) {
const cv::Mat& img = images[b];
if (img.empty()) {
_logger.LogFatal("ANSOYOLOV10OVOD::PreprocessingBatch",
"Empty image at index " + std::to_string(b), __FILE__, __LINE__);
return;
}
// Convert grayscale images to 3-channel BGR
cv::Mat processedImage;
if (img.channels() == 1) {
cv::cvtColor(img, processedImage, cv::COLOR_GRAY2BGR);
}
else {
processedImage = img;
}
// Store original dimensions
m_batchImgWidths[b] = processedImage.cols;
m_batchImgHeights[b] = processedImage.rows;
// Convert image to RGB
cv::Mat rgbImage;
cv::cvtColor(processedImage, rgbImage, cv::COLOR_BGR2RGB);
// Determine the maximum image length
int max_image_length = std::max(rgbImage.cols, rgbImage.rows);
// Create a square image with the maximum length and fill it with white
cv::Mat max_image = cv::Mat::ones(max_image_length, max_image_length, CV_8UC3) * 255;
// Copy the original image to the top-left corner of the square image
cv::Rect roi(0, 0, rgbImage.cols, rgbImage.rows);
rgbImage.copyTo(max_image(roi));
// Resize the image to the desired length
cv::Mat resizedImage;
cv::resize(max_image, resizedImage, cv::Size(length, length), 0.0f, 0.0f, cv::INTER_CUBIC);
// Calculate the scaling factor for this image
factors[b] = static_cast<float>(max_image_length) / static_cast<float>(length);
// Convert the image to float and normalize
resizedImage.convertTo(resizedImage, CV_32FC3, 1.0 / 255.0);
// Split channels and copy data to correct batch position
std::vector<cv::Mat> channels(3);
cv::split(resizedImage, channels);
// Calculate offset for this batch element
int batch_offset = b * 3 * length * length;
for (int c = 0; c < 3; ++c) {
std::memcpy(data.data() + batch_offset + c * length * length,
channels[c].data,
length * length * sizeof(float));
}
}
_logger.LogDebug("ANSOYOLOV10OVOD::PreprocessingBatch",
"Preprocessed batch of " + std::to_string(batchSize) + " images",
__FILE__, __LINE__);
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOYOLOV10OVOD::PreprocessingBatch", e.what(), __FILE__, __LINE__);
}
}
std::vector<Object> ANSOYOLOV10OVOD::PostProcessingBatch(float* result,
const std::vector<float>& factors,
int outputLength,
int batchSize,
const std::string& camera_id) {
std::vector<Object> allObjects;
try {
// Process each image's results in the batch
for (int b = 0; b < batchSize; ++b) {
std::vector<cv::Rect> position_boxes;
std::vector<int> class_ids;
std::vector<float> confidences;
// Reserve memory to avoid multiple reallocations
position_boxes.reserve(outputLength);
class_ids.reserve(outputLength);
confidences.reserve(outputLength);
// Get factor and dimensions for this specific image
float factor = factors[b];
int imgWidth = m_batchImgWidths[b];
int imgHeight = m_batchImgHeights[b];
// Calculate offset for this batch element's output
// Output format: [batch, num_detections, 6] where 6 = [x1, y1, x2, y2, conf, class]
int batch_offset = b * outputLength * 6;
for (int i = 0; i < outputLength; ++i) {
int s = batch_offset + 6 * i;
float confidence = result[s + 4];
if (confidence > _modelConfig.detectionScoreThreshold) {
float cx = result[s + 0];
float cy = result[s + 1];
float dx = result[s + 2];
float dy = result[s + 3];
int x = static_cast<int>(cx * factor);
int y = static_cast<int>(cy * factor);
int width = static_cast<int>((dx - cx) * factor);
int height = static_cast<int>((dy - cy) * factor);
// Clamp to image boundaries using batch-specific dimensions
x = std::max(0, x);
y = std::max(0, y);
width = std::min(imgWidth - x, width);
height = std::min(imgHeight - y, height);
cv::Rect box(x, y, width, height);
position_boxes.emplace_back(box);
class_ids.emplace_back(static_cast<int>(result[s + 5]));
confidences.emplace_back(confidence);
}
}
// Convert to Object format
int classNameSize = static_cast<int>(_classes.size());
for (size_t i = 0; i < position_boxes.size(); ++i) {
Object obj;
obj.classId = class_ids[i];
if (!_classes.empty()) {
if (obj.classId < classNameSize) {
obj.className = _classes[obj.classId];
}
else {
obj.className = _classes[classNameSize - 1];
}
}
else {
obj.className = "Unknown";
}
obj.confidence = confidences[i];
obj.box = position_boxes[i];
obj.polygon = ANSUtilityHelper::RectToNormalizedPolygon(obj.box, imgWidth, imgHeight);
obj.cameraId = camera_id;
allObjects.emplace_back(std::move(obj));
}
}
_logger.LogDebug("ANSOYOLOV10OVOD::PostProcessingBatch",
"Batch post-processing complete. Total detections: " + std::to_string(allObjects.size()),
__FILE__, __LINE__);
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOYOLOV10OVOD::PostProcessingBatch", e.what(), __FILE__, __LINE__);
}
return allObjects;
}
std::vector<Object> ANSOYOLOV10OVOD::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,
const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<Object> allObjects;
try {
if (inputImages.empty()) {
_logger.LogFatal("ANSOYOLOV10OVOD::DetectObjectsBatch", "Empty input images vector", __FILE__, __LINE__);
return allObjects;
}
int batchSize = static_cast<int>(inputImages.size());
// Validate batch size
if (batchSize > _modelConfig.gpuMaxBatchSize) {
_logger.LogError("ANSOYOLOV10OVOD::DetectObjectsBatch",
"Batch size " + std::to_string(batchSize) +
" exceeds configured maximum " + std::to_string(_modelConfig.gpuMaxBatchSize),
__FILE__, __LINE__);
return allObjects;
}
_logger.LogDebug("ANSOYOLOV10OVOD::DetectObjectsBatch",
"Processing batch of " + std::to_string(batchSize) + " images",
__FILE__, __LINE__);
// Preprocess all images
std::vector<float> factors;
std::vector<float> inputData;
PreprocessingBatch(inputImages, 640, factors, inputData);
if (inputData.empty()) {
_logger.LogFatal("ANSOYOLOV10OVOD::DetectObjectsBatch", "Preprocessing failed", __FILE__, __LINE__);
return allObjects;
}
// Update input tensor shape if batch size changed
auto current_shape = request.get_input_tensor().get_shape();
if (current_shape[0] != static_cast<size_t>(batchSize)) {
request.get_input_tensor().set_shape({
static_cast<size_t>(batchSize),
3,
640,
640
});
_logger.LogDebug("ANSOYOLOV10OVOD::DetectObjectsBatch",
"Input tensor reshaped to [" + std::to_string(batchSize) + ", 3, 640, 640]",
__FILE__, __LINE__);
}
// Copy input data to tensor
auto input_tensor = request.get_input_tensor();
std::memcpy(input_tensor.data<float>(), inputData.data(), inputData.size() * sizeof(float));
// Run inference
request.infer();
// Get output tensor
auto output_tensor = request.get_output_tensor();
float* output_data = output_tensor.data<float>();
// Get output shape to determine number of detections
auto output_shape = output_tensor.get_shape();
int outputLength = static_cast<int>(output_shape[1]); // Number of detections per image
// Post-process results
allObjects = PostProcessingBatch(output_data, factors, outputLength, batchSize, camera_id);
_logger.LogDebug("ANSOYOLOV10OVOD::DetectObjectsBatch",
"Batch processing complete. Total detections: " + std::to_string(allObjects.size()),
__FILE__, __LINE__);
return allObjects;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSOYOLOV10OVOD::DetectObjectsBatch", e.what(), __FILE__, __LINE__);
return allObjects;
}
}
}
//std::vector<Object> ANSOYOLOV10OVOD::RunInference(const cv::Mat& input, const std::string& camera_id) {
// std::lock_guard<std::recursive_mutex> lock(_mutex);
// std::vector<Object> output;
// // Validate license and initialization status
// if (!_licenseValid) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Invalid License", __FILE__, __LINE__);
// return output;
// }
// if (!_isInitialized) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Model is not initialized", __FILE__, __LINE__);
// return output;
// }
// try {
// // Validate input
// if (input.empty()) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Input frame is empty", __FILE__, __LINE__);
// return output;
// }
// if ((input.cols < 10) || (input.rows < 10)) return output;
// m_imgWidth = static_cast<float>(input.cols);
// m_imgHeight = static_cast<float>(input.rows);
// float factor = 0.0f;
// int imageSize = 640;
// cv::Mat frame = input;// input.clone(); // Clone the input to avoid modifying the original
// //int maxImageSize = std::max(frame.cols, frame.rows);
// //if (maxImageSize < imageSize)imageSize = maxImageSize;
// std::vector<float> inputData(imageSize * imageSize * 3);
// // Preprocess the input frame
// Preprocessing(&frame, imageSize, &factor, inputData);
// // Ensure input tensor is valid
// auto inputTensor = request.get_input_tensor();
// if (!inputTensor) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Failed to retrieve input tensor", __FILE__, __LINE__);
// frame.release();
// return output;
// }
// if (!inputTensor.data<float>()) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Input tensor data pointer is null", __FILE__, __LINE__);
// frame.release();
// return output;
// }
// // Ensure inputData is not empty before performing memcpy
// if (inputData.empty()) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Input data is empty", __FILE__, __LINE__);
// frame.release();
// return output;
// }
// // Copy input data into the tensor
// std::memcpy(inputTensor.data<float>(), inputData.data(), inputData.size() * sizeof(float));
// // Run inference
// if (!request) {
// request = compiled_model.create_infer_request();
// request.get_input_tensor().set_shape({ 1, 3, unsigned long(imageSize), unsigned long(imageSize) });
// }
// request.infer();
// // Retrieve output tensor
// auto outputTensor = request.get_output_tensor();
// if (!outputTensor) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Failed to retrieve output tensor", __FILE__, __LINE__);
// frame.release();
// return output;
// }
// // Get output data
// float* output_data = outputTensor.data<float>();
// if (!output_data) {
// _logger.LogError("ANSOYOLOV10OVOD::RunInference", "Output tensor data pointer is null", __FILE__, __LINE__);
// frame.release();
// return output;
// }
// // Post-process the output data
// output = PostProcessing(output_data, factor, 300, camera_id);
// frame.release();
// }
// catch (const std::exception& e) {
// _logger.LogFatal("ANSOYOLOV10OVOD::RunInference", e.what(), __FILE__, __LINE__);
// }
// catch (...) {
// _logger.LogFatal("ANSOYOLOV10OVOD::RunInference", "An unknown error occurred", __FILE__, __LINE__);
// }
// return output;
//}
+52
View File
@@ -0,0 +1,52 @@
#ifndef ANSOYOLOV10OVOD_H
#define ANSOYOLOV10OVOD_H
#pragma once
#include "ANSEngineCommon.h"
#include<openvino/openvino.hpp>
namespace ANSCENTER {
class ANSENGINE_API ANSOYOLOV10OVOD :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className,const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSOYOLOV10OVOD();
private:
std::string _modelFilePath;
float m_ratio = 1;
float m_imgWidth = 0;
float m_imgHeight = 0;
// ADD THESE for batch processing
std::vector<int> m_batchImgHeights;
std::vector<int> m_batchImgWidths;
std::vector<float> _inputData; // Reusable buffer for input tensor data
cv::Mat _frameBuffer; // Reusable buffer for preprocessing
private:
void InitialModel(const std::string& model_path);
void InitModelStaticBatchSize(const std::string& model_path);
void Preprocessing(cv::Mat* img, int length, float* factor, std::vector<float>& data);
cv::Mat resizeKeepAspectRatioPadRightBottom(const cv::Mat& input, size_t height, size_t width, const cv::Scalar& bgcolor);
std::vector<Object> PostProcessing(float* result, float factor, int outputLength, const std::string& camera_id);
std::vector<Object> PostProcessingBatch(float* result,
const std::vector<float>& factors,
int outputLength,
int batchSize,
const std::string& camera_id);
void PreprocessingBatch(const std::vector<cv::Mat>& images,
int length,
std::vector<float>& factors,
std::vector<float>& data);
std::vector<Object> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
ov::InferRequest request;
ov::CompiledModel compiled_model;
};
}
#endif
File diff suppressed because it is too large Load Diff
+93
View File
@@ -0,0 +1,93 @@
#ifndef ANSYOLOV10RTOD_H
#define ANSYOLOV10RTOD_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "NV12PreprocessHelper.h"
namespace ANSCENTER {
class ANSENGINE_API ANSYOLOV10RTOD :public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
virtual std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs, const std::string& camera_id) override;
bool Destroy();
~ANSYOLOV10RTOD();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
ANSCENTER::Options m_options;
private:
// Per-call batch metadata — passed by out-param to avoid shared state
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
// Per-call single-image metadata
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
// Preprocess the input; fills outMeta with per-call image geometry
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage,
ImageMetadata& outMeta);
// Postprocess the output
std::vector<Object> Postprocess(std::vector<float>& featureVector,
const std::string& camera_id,
const ImageMetadata& meta);
// Postprocess the output for segmentation model
std::vector<Object> PostProcessSegmentation(std::vector<std::vector<float>>& featureVectors,
const std::string& camera_id,
const ImageMetadata& meta);
// Postprocess the output for pose model
std::vector<Object> PostProcessPose(std::vector<float>& featureVector,
const std::string& camera_id,
const ImageMetadata& meta);
std::vector<Object> DetectObjects(const cv::Mat& inputImage, const std::string& camera_id);
std::vector<std::vector<Object>> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(
const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata);
std::vector<Object> PostprocessBatch(std::vector<float>& featureVector,
const std::string& camera_id,
size_t batchIdx,
const BatchMetadata& metadata);
NV12PreprocessHelper m_nv12Helper;
std::unique_ptr<Engine<float>> m_trtEngine = nullptr;
// Used for image preprocessing
// YoloV8 model expects values between [0.f, 1.f] so we use the following params
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
// Filter thresholds
float PROBABILITY_THRESHOLD;
float NMS_THRESHOLD;
int TOP_K;
// Segmentation constants
int SEG_CHANNELS;
int SEG_H;
int SEG_W;
float SEGMENTATION_THRESHOLD;
// Pose estimation constant
int NUM_KPS;
float KPS_THRESHOLD;
};
}
#endif
+935
View File
@@ -0,0 +1,935 @@
#include "ANSYOLOV12RTOD.h"
#include "Utility.h"
#include <opencv2/cudaimgproc.hpp>
#include <future>
namespace ANSCENTER
{
bool ANSYOLOV12RTOD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) {
return false;
}
if (!FileExist(_modelFilePath)) {
this->_logger.LogFatal("ANSYOLOV12RTOD::OptimizeModel", "Raw model file path does not exist", __FILE__, __LINE__);
return false;
}
try {
_fp16 = fp16;
optimizedModelFolder = GetParentFolder(_modelFilePath);
// Check if the engine already exists to avoid reinitializing
if (!m_trtEngine) {
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = optimizedModelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// Build the TensorRT engine
auto succ = m_trtEngine->buildWithRetry(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE);
if (!succ) {
const std::string errMsg =
"Error: Unable to build the TensorRT engine. "
"Try increasing TensorRT log severity to kVERBOSE.";
this->_logger.LogError("ANSYOLOV12RTOD::OptimizeModel", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
_modelLoadValid = true;
return true;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::OptimizeModel", e.what(), __FILE__, __LINE__);
optimizedModelFolder.clear();
return false;
}
}
bool ANSYOLOV12RTOD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
_fp16 = true; // Load Model from Here
// Load Model from Here
TOP_K = 100;
SEG_CHANNELS = 32;
PROBABILITY_THRESHOLD = _modelConfig.detectionScoreThreshold;
NMS_THRESHOLD = _modelConfig.modelMNSThreshold;
SEGMENTATION_THRESHOLD = 0.5f;
SEG_H = 160;
SEG_W = 160;
NUM_KPS = _modelConfig.numKPS;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
SEG_CHANNELS = 32; // For segmentation
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = _modelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSYOLOV12RTOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSYOLOV12RTOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// Load the TensorRT engine file
if (this->_loadEngineOnCreation) {
auto succ = m_trtEngine->buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
if (!succ) {
const std::string errMsg = "Error: Unable to load TensorRT engine weights into memory. " + _modelFilePath;
this->_logger.LogError("ANSYOLOV12RTOD::Initialize", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
}
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSYOLOV12RTOD::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName,std::string className, const std::string& modelFolder, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig,modelName, className,modelFolder, labelMap);
if (!result) return false;
std::string _modelName = modelName;
if (_modelName.empty()) {
_modelName = "train_last";
}
std::string modelFullName = _modelName + ".onnx";
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
_fp16 = true; // Load Model from Here
// Load Model from Here
TOP_K = 100;
SEG_CHANNELS = 32;
PROBABILITY_THRESHOLD = _modelConfig.detectionScoreThreshold;
NMS_THRESHOLD = _modelConfig.modelMNSThreshold;
SEGMENTATION_THRESHOLD = 0.5f;
SEG_H = 160;
SEG_W = 160;
NUM_KPS = _modelConfig.numKPS;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
SEG_CHANNELS = 32; // For segmentation
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.maxInputHeight = _modelConfig.maxInputHeight;
m_options.minInputHeight = _modelConfig.minInputHeight;
m_options.optInputHeight = _modelConfig.optInputHeight;
m_options.maxInputWidth = _modelConfig.maxInputWidth;
m_options.minInputWidth = _modelConfig.minInputWidth;
m_options.optInputWidth = _modelConfig.optInputWidth;
m_options.engineFileDir = _modelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
_classFilePath = CreateFilePath(_modelFolder, className);
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSYOLOV12RTOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSYOLOV12RTOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load the TensorRT engine file
if (this->_loadEngineOnCreation) {
auto succ = m_trtEngine->buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
if (!succ) {
const std::string errMsg = "Error: Unable to load TensorRT engine weights into memory. " + _modelFilePath;
this->_logger.LogError("ANSYOLOV12RTOD::Initialize", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
}
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSYOLOV12RTOD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
const bool engineAlreadyLoaded = _modelLoadValid && _isInitialized && m_trtEngine != nullptr;
_modelLoadValid = false;
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
// Parsing for YOLO only here
_modelConfig = modelConfig;
_modelConfig.detectionType = ANSCENTER::DetectionType::DETECTION;
_modelConfig.modelType = ModelType::TENSORRT;
_modelConfig.inpHeight = 640;
_modelConfig.inpWidth = 640;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) // 133 = COCO wholebody max
_modelConfig.numKPS = 17;
if (_modelConfig.kpsThreshold == 0)_modelConfig.kpsThreshold = 0.5; // If not define
// if (_modelConfig.precisionType == PrecisionType::FP16)_fp16 = true;
_fp16 = true; // Load Model from Here
// Load Model from Here
TOP_K = 100;
SEG_CHANNELS = 32;
PROBABILITY_THRESHOLD = _modelConfig.detectionScoreThreshold;
NMS_THRESHOLD = _modelConfig.modelMNSThreshold;
SEGMENTATION_THRESHOLD = 0.5f;
SEG_H = 160;
SEG_W = 160;
NUM_KPS = _modelConfig.numKPS;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
SEG_CHANNELS = 32; // For segmentation
if (!m_trtEngine) {
// Fixed batch size of 1 for this model
m_options.optBatchSize = _modelConfig.gpuOptBatchSize;
m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize;
m_options.deviceIndex = _modelConfig.gpuDeviceIndex;
m_options.engineFileDir = _modelFolder;
// Use FP16 or FP32 precision based on the input flag
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
// Create the TensorRT inference engine
m_trtEngine = std::make_unique<Engine<float>>(m_options);
}
// 0. Check if the configuration file exist
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0)_modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0)_modelConfig.inpWidth = inputShape[1];
}
}
else {// This is old version of model zip file
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValidFileName(_classFilePath);
if (!isValidFileName)
{
this->_logger.LogDebug("ANSYOLOV12RTOD::Initialize. Load classes from string", _classFilePath, __FILE__, __LINE__);
LoadClassesFromString();
}
else {
this->_logger.LogDebug("ANSYOLOV12RTOD::Initialize. Load classes from file", _classFilePath, __FILE__, __LINE__);
LoadClassesFromFile();
}
}
// 1. Load labelMap and engine
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
// Load the TensorRT engine file
if (this->_loadEngineOnCreation && !engineAlreadyLoaded) {
auto succ = m_trtEngine->buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
if (!succ) {
const std::string errMsg = "Error: Unable to load TensorRT engine weights into memory. " + _modelFilePath;
this->_logger.LogError("ANSYOLOV12RTOD::Initialize", errMsg, __FILE__, __LINE__);
_modelLoadValid = false;
return false;
}
}
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
std::vector<Object> ANSYOLOV12RTOD::RunInference(const cv::Mat& inputImgBGR) {
return RunInference(inputImgBGR, "TensorRT12Cam");
}
std::vector<Object> ANSYOLOV12RTOD::RunInference(const cv::Mat& inputImgBGR, const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
// Validate model, license, and initialization
if (!_modelLoadValid) {
this->_logger.LogFatal("ANSYOLOV12RTOD::RunInference",
"Cannot load the TensorRT model. Please check if it is exist", __FILE__, __LINE__);
return {};
}
if (!_licenseValid) {
this->_logger.LogFatal("ANSYOLOV12RTOD::RunInference",
"Runtime license is not valid or expired. Please contact ANSCENTER", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
this->_logger.LogFatal("ANSYOLOV12RTOD::RunInference",
"Model is not initialized", __FILE__, __LINE__);
return {};
}
// Validate input
if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) {
return {};
}
try {
return DetectObjects(inputImgBGR, camera_id);
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::RunInference", e.what(), __FILE__, __LINE__);
return {};
}
}
ANSYOLOV12RTOD::~ANSYOLOV12RTOD() {
try {
Destroy();
}
catch (std::exception& e) {
this->_logger.LogError("ANSYOLOV12RTOD::~ANSYOLOV12RTOD()", e.what(), __FILE__, __LINE__);
}
}
bool ANSYOLOV12RTOD::Destroy() {
try {
m_trtEngine.reset();
m_nv12Helper.destroy();
return true;
}
catch (std::exception& e) {
this->_logger.LogError("ANSYOLOV12RTOD::~ANSYOLOV12RTOD()", e.what(), __FILE__, __LINE__);
return false;
}
}
// private
std::vector<Object> ANSYOLOV12RTOD::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {
// Phase 1: Preprocess under brief lock — try NV12 fast path first
ImageMetadata meta;
std::vector<std::vector<cv::cuda::GpuMat>> input;
bool usedNV12 = false;
float bgrFullResScaleX = 1.0f, bgrFullResScaleY = 1.0f;
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
const int inferenceGpu = m_trtEngine ? m_trtEngine->getPreferredDeviceIndex() : 0;
const auto& inputDims = m_trtEngine->getInputDims();
const int inputW = inputDims[0].d[2];
const int inputH = inputDims[0].d[1];
auto nv12 = m_nv12Helper.tryNV12(inputImage, inferenceGpu, inputW, inputH,
NV12PreprocessHelper::defaultYOLOLauncher(),
_logger, "ANSYOLOV12RTOD");
if (nv12.succeeded) {
meta.imgWidth = nv12.metaWidth;
meta.imgHeight = nv12.metaHeight;
meta.ratio = nv12.ratio;
input = {{ std::move(nv12.gpuRGB) }};
usedNV12 = true;
}
else if (nv12.useBgrFullRes) {
input = Preprocess(nv12.bgrFullResImg, meta);
usedNV12 = !input.empty();
bgrFullResScaleX = nv12.bgrFullResScaleX;
bgrFullResScaleY = nv12.bgrFullResScaleY;
}
if (input.empty()) {
input = Preprocess(inputImage, meta);
}
m_nv12Helper.tickInference();
}
if (input.empty()) return {};
// Phase 2: Inference - mutex released; pool dispatches to idle GPU slot
std::vector<std::vector<std::vector<float>>> featureVectors;
auto succ = m_trtEngine->runInference(input, featureVectors);
if (!succ) {
this->_logger.LogFatal("ANSYOLOV12RTOD::DetectObjects", "Error running inference", __FILE__, __LINE__);
return {};
}
// Phase 3: Postprocess under brief lock
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<float> featureVector;
Engine<float>::transformOutput(featureVectors, featureVector);
auto ret = Postprocess(featureVector, camera_id, meta);
// Rescale coords from full-res to display-res (BGR full-res path)
if (bgrFullResScaleX != 1.0f || bgrFullResScaleY != 1.0f) {
for (auto& obj : ret) {
obj.box.x = static_cast<int>(obj.box.x * bgrFullResScaleX);
obj.box.y = static_cast<int>(obj.box.y * bgrFullResScaleY);
obj.box.width = static_cast<int>(obj.box.width * bgrFullResScaleX);
obj.box.height = static_cast<int>(obj.box.height * bgrFullResScaleY);
for (auto& pt : obj.polygon) {
pt.x *= bgrFullResScaleX;
pt.y *= bgrFullResScaleY;
}
for (size_t k = 0; k + 2 < obj.kps.size(); k += 3) {
obj.kps[k] *= bgrFullResScaleX;
obj.kps[k + 1] *= bgrFullResScaleY;
}
}
}
if (_trackerEnabled) {
ret = ApplyTracking(ret, camera_id);
if (_stabilizationEnabled) ret = StabilizeDetections(ret, camera_id);
}
return ret;
}
std::vector<std::vector<cv::cuda::GpuMat>> ANSYOLOV12RTOD::Preprocess(const cv::Mat& inputImage, ImageMetadata& outMeta) {
try {
if (!_licenseValid) {
this->_logger.LogFatal("ANSYOLOV12RTOD::Preprocess", "Invalid license", __FILE__, __LINE__);
return {};
}
const auto& inputDims = m_trtEngine->getInputDims();
cv::cuda::Stream stream;
cv::cuda::GpuMat img;
// Upload to GPU
if (inputImage.channels() == 1) {
cv::Mat img3Channel;
cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);
img.upload(img3Channel, stream);
}
else {
img.upload(inputImage, stream);
}
// Convert BGR to RGB
cv::cuda::GpuMat imgRGB;
cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);
stream.waitForCompletion();
outMeta.imgHeight = imgRGB.rows;
outMeta.imgWidth = imgRGB.cols;
if (outMeta.imgHeight > 0 && outMeta.imgWidth > 0) {
outMeta.ratio = 1.f / std::min(inputDims[0].d[2] / static_cast<float>(imgRGB.cols),
inputDims[0].d[1] / static_cast<float>(imgRGB.rows));
cv::cuda::GpuMat resized = imgRGB;
// Resize to the model's expected input size while maintaining aspect ratio with padding
if (resized.rows != inputDims[0].d[1] || resized.cols != inputDims[0].d[2]) {
resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(imgRGB, inputDims[0].d[1], inputDims[0].d[2]);
}
// Convert to format expected by our inference engine
std::vector<cv::cuda::GpuMat> input{ std::move(resized) };
std::vector<std::vector<cv::cuda::GpuMat>> inputs{ std::move(input) };
return inputs;
}
else {
this->_logger.LogFatal("TENSORRTCL::Preprocess",
"Image height or width is zero after processing (Width: " + std::to_string(outMeta.imgWidth) +
", Height: " + std::to_string(outMeta.imgHeight) + ")",
__FILE__, __LINE__);
return {};
}
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::Preprocess", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<Object> ANSYOLOV12RTOD::Postprocess(std::vector<float>& featureVector, const std::string& camera_id, const ImageMetadata& meta) {
try {
const auto& outputDims = m_trtEngine->getOutputDims();
auto numChannels = outputDims[0].d[1];// similar to detection_attribute_size
auto numAnchors = outputDims[0].d[2]; // similar to num_detections
auto numClasses = _classes.size();
std::vector<cv::Rect> bboxes;
std::vector<float> scores;
std::vector<int> labels;
std::vector<int> indices;
cv::Mat output = cv::Mat(numChannels, numAnchors, CV_32F, featureVector.data());
output = output.t();
// Get all the YOLO proposals
for (int i = 0; i < numAnchors; i++) {
auto rowPtr = output.row(i).ptr<float>();
auto bboxesPtr = rowPtr;
auto scoresPtr = rowPtr + 4;
auto maxSPtr = std::max_element(scoresPtr, scoresPtr + numClasses);
float score = *maxSPtr;
if (score > this->_modelConfig.detectionScoreThreshold) {
float x = *bboxesPtr++;
float y = *bboxesPtr++;
float w = *bboxesPtr++;
float h = *bboxesPtr;
float x0 = std::clamp((x - 0.5f * w) * meta.ratio, 0.f, meta.imgWidth);
float y0 = std::clamp((y - 0.5f * h) * meta.ratio, 0.f, meta.imgHeight);
float x1 = std::clamp((x + 0.5f * w) * meta.ratio, 0.f, meta.imgWidth);
float y1 = std::clamp((y + 0.5f * h) * meta.ratio, 0.f, meta.imgHeight);
int label = maxSPtr - scoresPtr;
cv::Rect_<float> bbox;
bbox.x = x0;
bbox.y = y0;
bbox.width = x1 - x0;
bbox.height = y1 - y0;
bbox.x = std::clamp(bbox.x, 0.f, meta.imgWidth);
bbox.y = std::clamp(bbox.y, 0.f, meta.imgHeight);
bbox.width = std::clamp(bbox.width, 0.f, meta.imgWidth);
bbox.height = std::clamp(bbox.height, 0.f, meta.imgHeight);
bboxes.push_back(bbox);
labels.push_back(label);
scores.push_back(score);
}
}
// Run NMS
cv::dnn::NMSBoxesBatched(bboxes, scores, labels, PROBABILITY_THRESHOLD, NMS_THRESHOLD, indices);
int classNameSize = static_cast<int>(_classes.size());
std::vector<Object> objects;
// Choose the top k detections
for (auto& chosenIdx : indices) {
if (scores[chosenIdx] > _modelConfig.detectionScoreThreshold) {
Object obj{};
obj.confidence = scores[chosenIdx];
obj.classId = labels[chosenIdx];
obj.box = bboxes[chosenIdx];
obj.polygon = ANSUtilityHelper::RectToNormalizedPolygon(obj.box, meta.imgWidth, meta.imgHeight);
if (!_classes.empty()) {
if (obj.classId < classNameSize) {
obj.className = _classes[obj.classId];
}
else {
obj.className = _classes[classNameSize - 1]; // Use last valid class name if out of range
}
}
else {
obj.className = "Unknown"; // Fallback if _classes is empty
}
obj.cameraId = camera_id;
objects.push_back(obj);
}
}
//EnqueueDetection(objects, camera_id);
return objects;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSYOLOV12RTOD::Postproces", e.what(), __FILE__, __LINE__);
std::vector<Object> result;
result.clear();
return result;
}
}
std::vector<std::vector<Object>> ANSYOLOV12RTOD::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id) {
// Validate under brief lock
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (inputImages.empty()) {
_logger.LogFatal("ANSYOLOV12RTOD::DetectObjectsBatch", "Empty input images vector", __FILE__, __LINE__);
return {};
}
}
// Auto-split if batch exceeds engine capacity
const int maxBatch = m_options.maxBatchSize > 0 ? m_options.maxBatchSize : 1;
if (static_cast<int>(inputImages.size()) > maxBatch) {
const size_t numImages = inputImages.size();
std::vector<std::vector<Object>> allResults;
allResults.reserve(numImages);
// Process chunks sequentially to avoid GPU contention on the same engine
for (size_t start = 0; start < numImages; start += static_cast<size_t>(maxBatch)) {
const size_t end = std::min(start + static_cast<size_t>(maxBatch), numImages);
std::vector<cv::Mat> chunk(inputImages.begin() + start, inputImages.begin() + end);
auto chunkResults = DetectObjectsBatch(chunk, camera_id);
if (chunkResults.size() == chunk.size()) {
for (auto& r : chunkResults) allResults.push_back(std::move(r));
}
else {
_logger.LogError("ANSYOLOV12RTOD::DetectObjectsBatch",
"Chunk returned " + std::to_string(chunkResults.size()) +
" results, expected " + std::to_string(chunk.size()) +
". Padding with empty results.", __FILE__, __LINE__);
for (auto& r : chunkResults) allResults.push_back(std::move(r));
for (size_t pad = chunkResults.size(); pad < chunk.size(); ++pad) {
allResults.push_back({});
}
}
}
return allResults;
}
_logger.LogDebug("ANSYOLOV12RTOD::DetectObjectsBatch",
"Processing batch of " + std::to_string(inputImages.size()) + " images",
__FILE__, __LINE__);
// Phase 1: Preprocess under brief lock
BatchMetadata metadata;
std::vector<std::vector<cv::cuda::GpuMat>> inputs;
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
inputs = PreprocessBatch(inputImages, metadata);
}
if (inputs.empty() || inputs[0].empty()) {
_logger.LogFatal("ANSYOLOV12RTOD::DetectObjectsBatch", "Preprocessing failed", __FILE__, __LINE__);
return {};
}
// Phase 2: Inference - mutex released; pool dispatches to idle GPU slot
std::vector<std::vector<std::vector<float>>> featureVectors;
auto succ = m_trtEngine->runInference(inputs, featureVectors);
if (!succ) {
_logger.LogFatal("ANSYOLOV12RTOD::DetectObjectsBatch", "Error running inference", __FILE__, __LINE__);
return {};
}
// Phase 3: Parallel postprocessing - each image is independent
const size_t numBatch = featureVectors.size();
std::vector<std::vector<Object>> batchDetections(numBatch);
std::vector<std::future<std::vector<Object>>> postFutures;
postFutures.reserve(numBatch);
for (size_t batchIdx = 0; batchIdx < numBatch; ++batchIdx) {
const auto& batchOutput = featureVectors[batchIdx];
std::vector<float> featureVector =
batchOutput.empty() ? std::vector<float>{} : batchOutput[0];
postFutures.push_back(std::async(std::launch::async,
[this, fv = std::move(featureVector), cid = camera_id,
idx = batchIdx, &metadata]() mutable {
return PostprocessBatch(fv, cid, idx, metadata);
}));
}
// Gather results in original order; metadata stays alive until all futures joined
for (size_t i = 0; i < numBatch; ++i)
batchDetections[i] = postFutures[i].get();
_logger.LogDebug("ANSYOLOV12RTOD::DetectObjectsBatch",
"Batch processing complete. Images: " + std::to_string(numBatch),
__FILE__, __LINE__);
return batchDetections;
}
std::vector<std::vector<cv::cuda::GpuMat>> ANSYOLOV12RTOD::PreprocessBatch(const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata) {
try {
if (!_licenseValid) {
_logger.LogFatal("ANSYOLOV12RTOD::PreprocessBatch", "Invalid license", __FILE__, __LINE__);
return {};
}
const auto& inputDims = m_trtEngine->getInputDims();
const int inputH = inputDims[0].d[1];
const int inputW = inputDims[0].d[2];
// Store original image dimensions for each image in batch
outMetadata.imgHeights.resize(inputImages.size());
outMetadata.imgWidths.resize(inputImages.size());
outMetadata.ratios.resize(inputImages.size());
std::vector<cv::cuda::GpuMat> batchProcessed;
batchProcessed.reserve(inputImages.size());
cv::cuda::Stream stream;
// Process each image
for (size_t i = 0; i < inputImages.size(); ++i) {
const auto& inputImage = inputImages[i];
if (inputImage.empty()) {
_logger.LogFatal("ANSYOLOV12RTOD::PreprocessBatch",
"Empty input image at index " + std::to_string(i), __FILE__, __LINE__);
return {};
}
// Upload to GPU
cv::cuda::GpuMat img;
// Convert grayscale to BGR if needed
if (inputImage.channels() == 1) {
cv::Mat img3Channel;
cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);
img.upload(img3Channel, stream);
}
else {
img.upload(inputImage, stream);
}
// Convert BGR to RGB
cv::cuda::GpuMat imgRGB;
cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);
// Store original dimensions
outMetadata.imgHeights[i] = imgRGB.rows;
outMetadata.imgWidths[i] = imgRGB.cols;
if (outMetadata.imgHeights[i] <= 0 || outMetadata.imgWidths[i] <= 0) {
_logger.LogFatal("ANSYOLOV12RTOD::PreprocessBatch",
"Image " + std::to_string(i) + " has invalid dimensions (Width: " +
std::to_string(outMetadata.imgWidths[i]) + ", Height: " +
std::to_string(outMetadata.imgHeights[i]) + ")",
__FILE__, __LINE__);
return {};
}
// Calculate ratio for this image
outMetadata.ratios[i] = 1.f / std::min(inputW / static_cast<float>(imgRGB.cols),
inputH / static_cast<float>(imgRGB.rows));
// Resize with padding
cv::cuda::GpuMat resized = imgRGB;
if (resized.rows != inputH || resized.cols != inputW) {
resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(imgRGB, inputH, inputW);
}
batchProcessed.push_back(std::move(resized));
}
stream.waitForCompletion();
// Return as required format
std::vector<std::vector<cv::cuda::GpuMat>> inputs;
inputs.push_back(std::move(batchProcessed));
return inputs;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSYOLOV12RTOD::PreprocessBatch", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<Object> ANSYOLOV12RTOD::PostprocessBatch(std::vector<float>& featureVector,
const std::string& camera_id,
size_t batchIdx,
const BatchMetadata& metadata) {
try {
const auto& outputDims = m_trtEngine->getOutputDims();
auto numChannels = outputDims[0].d[1]; // detection_attribute_size
auto numAnchors = outputDims[0].d[2]; // num_detections
auto numClasses = _classes.size();
// Get the ratio and dimensions for this specific image in the batch
float ratio = metadata.ratios[batchIdx];
int imgWidth = metadata.imgWidths[batchIdx];
int imgHeight = metadata.imgHeights[batchIdx];
std::vector<cv::Rect> bboxes;
std::vector<float> scores;
std::vector<int> labels;
std::vector<int> indices;
cv::Mat output = cv::Mat(numChannels, numAnchors, CV_32F, featureVector.data());
output = output.t();
// Get all the YOLO proposals
for (int i = 0; i < numAnchors; i++) {
auto rowPtr = output.row(i).ptr<float>();
auto bboxesPtr = rowPtr;
auto scoresPtr = rowPtr + 4;
auto maxSPtr = std::max_element(scoresPtr, scoresPtr + numClasses);
float score = *maxSPtr;
if (score > _modelConfig.detectionScoreThreshold) {
float x = *bboxesPtr++;
float y = *bboxesPtr++;
float w = *bboxesPtr++;
float h = *bboxesPtr;
// Use batch-specific ratio and dimensions
float x0 = std::clamp((x - 0.5f * w) * ratio, 0.f, static_cast<float>(imgWidth));
float y0 = std::clamp((y - 0.5f * h) * ratio, 0.f, static_cast<float>(imgHeight));
float x1 = std::clamp((x + 0.5f * w) * ratio, 0.f, static_cast<float>(imgWidth));
float y1 = std::clamp((y + 0.5f * h) * ratio, 0.f, static_cast<float>(imgHeight));
int label = maxSPtr - scoresPtr;
cv::Rect_<float> bbox;
bbox.x = x0;
bbox.y = y0;
bbox.width = x1 - x0;
bbox.height = y1 - y0;
// Clamp bbox to image boundaries
bbox.x = std::clamp(bbox.x, 0.f, static_cast<float>(imgWidth));
bbox.y = std::clamp(bbox.y, 0.f, static_cast<float>(imgHeight));
bbox.width = std::clamp(bbox.width, 0.f, static_cast<float>(imgWidth));
bbox.height = std::clamp(bbox.height, 0.f, static_cast<float>(imgHeight));
bboxes.push_back(bbox);
labels.push_back(label);
scores.push_back(score);
}
}
// Run NMS
cv::dnn::NMSBoxesBatched(bboxes, scores, labels, PROBABILITY_THRESHOLD, NMS_THRESHOLD, indices);
int classNameSize = static_cast<int>(_classes.size());
std::vector<Object> objects;
// Choose the top k detections
for (auto& chosenIdx : indices) {
if (scores[chosenIdx] > _modelConfig.detectionScoreThreshold) {
Object obj{};
obj.confidence = scores[chosenIdx];
obj.classId = labels[chosenIdx];
obj.box = bboxes[chosenIdx];
obj.polygon = ANSUtilityHelper::RectToNormalizedPolygon(obj.box, imgWidth, imgHeight);
if (!_classes.empty()) {
if (obj.classId < classNameSize) {
obj.className = _classes[obj.classId];
}
else {
obj.className = _classes[classNameSize - 1];
}
}
else {
obj.className = "Unknown";
}
obj.cameraId = camera_id;
objects.push_back(obj);
}
}
return objects;
}
catch (std::exception& e) {
_logger.LogFatal("ANSYOLOV12RTOD::PostprocessBatch", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<std::vector<Object>> ANSYOLOV12RTOD::RunInferencesBatch(
const std::vector<cv::Mat>& inputs, const std::string& camera_id)
{
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!_modelLoadValid) {
_logger.LogError("ANSYOLOV12RTOD::RunInferencesBatch", "Model not loaded", __FILE__, __LINE__);
return {};
}
if (!_licenseValid) {
_logger.LogError("ANSYOLOV12RTOD::RunInferencesBatch", "Invalid license", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("ANSYOLOV12RTOD::RunInferencesBatch", "Engine not initialized", __FILE__, __LINE__);
return {};
}
if (inputs.empty()) return {};
}
try {
return DetectObjectsBatch(inputs, camera_id);
}
catch (const std::exception& e) {
_logger.LogFatal("ANSYOLOV12RTOD::RunInferencesBatch", e.what(), __FILE__, __LINE__);
return {};
}
}
}
+86
View File
@@ -0,0 +1,86 @@
#ifndef ANSYOLOV12RTOD_H
#define ANSYOLOV12RTOD_H
#pragma once
#include "ANSEngineCommon.h"
#include "engine.h"
#include "NV12PreprocessHelper.h"
namespace ANSCENTER {
class ANSENGINE_API ANSYOLOV12RTOD :public ANSODBase
{
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
virtual bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
virtual std::vector<std::vector<Object>> RunInferencesBatch(
const std::vector<cv::Mat>& inputs, const std::string& camera_id) override;
bool Destroy();
~ANSYOLOV12RTOD();
private:
std::string _modelFilePath;
bool _modelLoadValid;
bool _fp16{ false };
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
private:
// Per-call batch metadata — passed by out-param to avoid shared state
struct BatchMetadata {
std::vector<int> imgHeights;
std::vector<int> imgWidths;
std::vector<float> ratios;
};
// Per-call single-image metadata
struct ImageMetadata {
float ratio = 1.f;
float imgWidth = 0.f;
float imgHeight = 0.f;
};
// Preprocess the input; fills outMeta with per-call image geometry
std::vector<std::vector<cv::cuda::GpuMat>> Preprocess(const cv::Mat& inputImage,
ImageMetadata& outMeta);
// Postprocess the output
std::vector<Object> Postprocess(std::vector<float>& featureVector,
const std::string& camera_id,
const ImageMetadata& meta);
std::vector<Object> DetectObjects(const cv::Mat& inputImage, const std::string& camera_id);
std::vector<std::vector<Object>> DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,
const std::string& camera_id);
std::vector<std::vector<cv::cuda::GpuMat>> PreprocessBatch(
const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata);
std::vector<Object> PostprocessBatch(std::vector<float>& featureVector,
const std::string& camera_id,
size_t batchIdx,
const BatchMetadata& metadata);
NV12PreprocessHelper m_nv12Helper;
std::unique_ptr<Engine<float>> m_trtEngine = nullptr;
// Used for image preprocessing
// YoloV8 model expects values between [0.f, 1.f] so we use the following params
const std::array<float, 3> SUB_VALS{ 0.f, 0.f, 0.f };
const std::array<float, 3> DIV_VALS{ 1.f, 1.f, 1.f };
const bool NORMALIZE = true;
ANSCENTER::Options m_options;
// Filter thresholds
float PROBABILITY_THRESHOLD;
float NMS_THRESHOLD;
int TOP_K;
// Segmentation constants
int SEG_CHANNELS;
int SEG_H;
int SEG_W;
float SEGMENTATION_THRESHOLD;
// Pose estimation constant
int NUM_KPS;
float KPS_THRESHOLD;
};
}
#endif
+152
View File
@@ -0,0 +1,152 @@
# ANSODEngine — Object Detection Engine DLL (CUDA + ONNX + TensorRT + OpenVINO)
# Source list matches original ANSODEngine.vcxproj exactly
set(ANSOD_SOURCES
ANSODEngine.cpp
ANSODHUB.cpp
ANSEngineCommon.cpp
ANSMotionDetector.cpp
ANSANOMALIB.cpp
ANSCLIPTokenizer.cpp
ANSCUSTOMDetector.cpp
ANSCUSTOMPY.cpp
ANSFD.cpp
ANSFaceDetectorEngine.cpp
ANSFaceRecognizerEngine.cpp
ANSGpuFrameRegistry.cpp
ANSONNXCL.cpp
ANSONNXOBB.cpp
ANSONNXPOSE.cpp
ANSONNXSAM3.cpp
ANSONNXSEG.cpp
ANSONNXYOLO.cpp
ANSOPENVINOCL.cpp
ANSOPENVINOOD.cpp
ANSOVFaceDetector.cpp
ANSOVSEG.cpp
ANSPOSE.cpp
ANSRTYOLO.cpp
ANSSAM.cpp
ANSSAM3.cpp
ANSTENSORRTCL.cpp
ANSTENSORRTPOSE.cpp
ANSTENSORRTSEG.cpp
ANSTENSORTRTOD.cpp
ANSYOLO12OD.cpp
ANSYOLOOD.cpp
ANSYOLOV10OVOD.cpp
ANSYOLOV10RTOD.cpp
ANSYOLOV12RTOD.cpp
Movienet.cpp
NV12PreprocessHelper.cpp
SCRFDFaceDetector.cpp
engine.cpp
dllmain.cpp
pch.cpp
)
file(GLOB ANSOD_HEADERS "*.h")
file(GLOB ANSOD_CUDA_HEADERS "CUDA/*.h" "CUDA/*.cuh")
file(GLOB ANSOD_CUDA_SOURCES "CUDA/*.cu" "*.cu")
# Engine wrapper sources from original vcxproj
set(ENGINE_SOURCES
${CMAKE_SOURCE_DIR}/engines/ONNXEngine/ONNXEngine.cpp
${CMAKE_SOURCE_DIR}/engines/ONNXEngine/ONNXSAM3.cpp
)
# OpenVINO engine sources from original vcxproj
set(OPENVINO_ENGINE_SOURCES
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/faceapp/cnn.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/detection_model.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/detection_model_faceboxes.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/detection_model_ssd.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/hpe_model_openpose.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/image_model.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/jpeg_restoration_model.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/model_base.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/openpose_decoder.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/models/super_resolution_model.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/pipelines/async_pipeline.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/pipelines/requests_pool.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/utils/args_helper.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/utils/common.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/utils/config_factory.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/utils/image_utils.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/utils/performance_metrics.cpp
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/src/utils/visualizer.cpp
)
add_library(ANSODEngine SHARED
${ANSOD_SOURCES}
${ANSOD_HEADERS}
${ANSOD_CUDA_HEADERS}
${ANSOD_CUDA_SOURCES}
${ENGINE_SOURCES}
${OPENVINO_ENGINE_SOURCES}
)
target_include_directories(ANSODEngine PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/CUDA
)
target_include_directories(ANSODEngine PRIVATE
${CMAKE_SOURCE_DIR}/engines/ONNXEngine
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/include
${CMAKE_SOURCE_DIR}/engines/OpenVINOEngine/include/faceapp
${CMAKE_SOURCE_DIR}/engines/TensorRTAPI/include
${CMAKE_SOURCE_DIR}/engines/TensorRTEngine
${CMAKE_SOURCE_DIR}/core/ANSLicensingSystem
${ANSLIBS_DIR}/nlohmann
${RESEARCH_DIR}/spdlog-1.12.0/include
${RESEARCH_DIR}
${RESEARCH_DIR}/ANS-HWiNFO/include
${ANLS_ROOT}/ODHUBYOLODarknet/include
${ANLS_ROOT}/TurboJpeg
C:/fastdeploy_gpu/include
${SHARED_INCLUDE_DIR}
)
target_link_libraries(ANSODEngine
PUBLIC ANSLibsLoader
PRIVATE ANSLicensingSystem
PUBLIC opencv
PRIVATE ANSMOT
PRIVATE onnxruntime
PRIVATE tensorrt
PRIVATE openvino
PRIVATE python311
PRIVATE turbojpeg
PRIVATE boost
PRIVATE CUDA::cudart_static
PRIVATE CUDA::cublas
PRIVATE CUDA::cublasLt
)
# Additional link dependencies from original vcxproj
target_link_directories(ANSODEngine PRIVATE
${ANLS_ROOT}/ODHUBYOLODarknet/lib
)
target_link_libraries(ANSODEngine PRIVATE
odhublib.lib odhublib_cpu.lib
nvinfer_10.lib nvonnxparser_10.lib
labview
ANSLicensingSystem
spdlog_dep
)
target_compile_definitions(ANSODEngine PRIVATE UNICODE _UNICODE NOMINMAX
ANSODENGINE_EXPORTS
ENGINE_EXPORTS
ORT_API_MANUAL_INIT
_USRDLL
)
if(WIN32)
target_link_libraries(ANSODEngine PRIVATE ${WIN_COMMON_LIBS})
else()
target_link_libraries(ANSODEngine PRIVATE ${UNIX_COMMON_LIBS})
endif()
target_precompile_headers(ANSODEngine PRIVATE pch.h)
File diff suppressed because it is too large Load Diff
File diff suppressed because one or more lines are too long
+17
View File
@@ -0,0 +1,17 @@
#ifndef TRTX_CUDA_UTILS_H_
#define TRTX_CUDA_UTILS_H_
#include <cuda_runtime_api.h>
#ifndef CUDA_CHECK
#define CUDA_CHECK(callstr)\
{\
cudaError_t error_code = callstr;\
if (error_code != cudaSuccess) {\
std::cerr << "CUDA error " << error_code << " at " << __FILE__ << ":" << __LINE__;\
assert(0);\
}\
}
#endif // CUDA_CHECK
#endif // TRTX_CUDA_UTILS_H_
+166
View File
@@ -0,0 +1,166 @@
#include "preprocess.h"
#include "cuda_utils.h"
#include "device_launch_parameters.h"
#include <iostream>
// Static buffers
static uint8_t* img_buffer_host = nullptr;
static uint8_t* img_buffer_device = nullptr;
struct AffineMatrix {
float value[6];
};
// CUDA error checking macro
#define CUDA_CALL(x) do { \
cudaError_t err = x; \
if (err != cudaSuccess) { \
std::cerr << "CUDA Error: " << cudaGetErrorString(err) \
<< " at " << __FILE__ << ":" << __LINE__ << std::endl; \
std::exit(EXIT_FAILURE); \
} \
} while (0)
// Kernel with logs
__global__ void warpaffine_kernel(
uint8_t* src, int src_line_size, int src_width,
int src_height, float* dst, int dst_width,
int dst_height, uint8_t const_value_st,
AffineMatrix d2s, int edge) {
int position = blockDim.x * blockIdx.x + threadIdx.x;
if (position >= edge) return;
int dx = position % dst_width;
int dy = position / dst_width;
// Transform source coordinates
float src_x = d2s.value[0] * dx + d2s.value[1] * dy + d2s.value[2] + 0.5f;
float src_y = d2s.value[3] * dx + d2s.value[4] * dy + d2s.value[5] + 0.5f;
printf("Thread %d: (dx, dy) = (%d, %d), (src_x, src_y) = (%.2f, %.2f)\n",
position, dx, dy, src_x, src_y);
float c0, c1, c2;
// Check if source coordinates are out of bounds
if (src_x < 0 || src_x >= src_width || src_y < 0 || src_y >= src_height) {
c0 = c1 = c2 = const_value_st; // Default value for out-of-range
} else {
int x_low = floorf(src_x);
int y_low = floorf(src_y);
int x_high = x_low + 1;
int y_high = y_low + 1;
// Handle boundary conditions
uint8_t* v1 = src + y_low * src_line_size + x_low * 3;
uint8_t* v2 = (x_high < src_width) ? src + y_low * src_line_size + x_high * 3 : v1;
uint8_t* v3 = (y_high < src_height) ? src + y_high * src_line_size + x_low * 3 : v1;
uint8_t* v4 = (x_high < src_width && y_high < src_height) ? src + y_high * src_line_size + x_high * 3 : v1;
// Bilinear interpolation weights
float lx = src_x - x_low;
float ly = src_y - y_low;
float hx = 1 - lx;
float hy = 1 - ly;
float w1 = hx * hy, w2 = lx * hy, w3 = hx * ly, w4 = lx * ly;
// Compute final pixel values
c0 = w1 * v1[0] + w2 * v2[0] + w3 * v3[0] + w4 * v4[0];
c1 = w1 * v1[1] + w2 * v2[1] + w3 * v3[1] + w4 * v4[1];
c2 = w1 * v1[2] + w2 * v2[2] + w3 * v3[2] + w4 * v4[2];
}
// Convert BGR to RGB
float tmp = c0; c0 = c2; c2 = tmp;
// Normalize pixel values
c0 /= 255.0f;
c1 /= 255.0f;
c2 /= 255.0f;
int area = dst_width * dst_height;
float* pdst_c0 = dst + dy * dst_width + dx;
float* pdst_c1 = pdst_c0 + area;
float* pdst_c2 = pdst_c1 + area;
// Store the normalized values
*pdst_c0 = c0;
*pdst_c1 = c1;
*pdst_c2 = c2;
}
// Host-side preprocessing function
void cuda_preprocess(
uint8_t* src, int src_width, int src_height,
float* dst, int dst_width, int dst_height,
cudaStream_t stream) {
int img_size = src_width * src_height * 3;
if (img_buffer_host == nullptr) {
std::cerr << "Error: img_buffer_host not allocated!" << std::endl;
}
if (src == nullptr) {
std::cerr << "Error: Source image pointer is null!" << std::endl;
}
size_t free_mem, total_mem;
cudaMemGetInfo(&free_mem, &total_mem);
std::cout << "Free GPU memory: " << free_mem << ", Total GPU memory: " << total_mem << std::endl;
cudaDeviceSynchronize();
std::cout << "Synchronized CUDA device." << std::endl;
// Copy data to pinned memory
std::cout << "Copying data to pinned memory..." << std::endl;
memcpy(img_buffer_host, src, img_size);
// Copy data to device memory
std::cout << "Copying data to device memory..." << std::endl;
CUDA_CALL(cudaMemcpyAsync(img_buffer_device, img_buffer_host, img_size, cudaMemcpyHostToDevice, stream));
CUDA_CALL(cudaStreamSynchronize(stream));
// Prepare the affine matrices
AffineMatrix s2d, d2s;
float scale = std::min(dst_height / (float)src_height, dst_width / (float)src_width);
s2d.value[0] = scale; s2d.value[1] = 0;
s2d.value[2] = -scale * src_width * 0.5 + dst_width * 0.5;
s2d.value[3] = 0; s2d.value[4] = scale;
s2d.value[5] = -scale * src_height * 0.5 + dst_height * 0.5;
cv::Mat m2x3_s2d(2, 3, CV_32F, s2d.value);
cv::Mat m2x3_d2s(2, 3, CV_32F, d2s.value);
cv::invertAffineTransform(m2x3_s2d, m2x3_d2s);
memcpy(d2s.value, m2x3_d2s.ptr<float>(0), sizeof(d2s.value));
int jobs = dst_width * dst_height;
int threads = 256;
int blocks = (jobs + threads - 1) / threads;
// Launch the kernel
std::cout << "Launching kernel..." << std::endl;
warpaffine_kernel<<<blocks, threads, 0, stream>>>(
img_buffer_device, src_width * 3, src_width, src_height,
dst, dst_width, dst_height, 128, d2s, jobs);
// Synchronize and check for errors
CUDA_CALL(cudaStreamSynchronize(stream));
std::cout << "Kernel execution completed." << std::endl;
}
void cuda_preprocess_init(int max_image_size) {
std::cout << "I am in the preprocess init" << std::endl;
CUDA_CALL(cudaMallocHost((void**)&img_buffer_host, max_image_size * 3));
CUDA_CALL(cudaMalloc((void**)&img_buffer_device, max_image_size * 3));
}
void cuda_preprocess_destroy() {
CUDA_CALL(cudaFree(img_buffer_device));
CUDA_CALL(cudaFreeHost(img_buffer_host));
}
+11
View File
@@ -0,0 +1,11 @@
#pragma once
#include <cuda_runtime.h>
#include <cstdint>
#include <opencv2/opencv.hpp>
void cuda_preprocess_init(int max_image_size);
void cuda_preprocess_destroy();
void cuda_preprocess(uint8_t* src, int src_width, int src_height,
float* dst, int dst_width, int dst_height,
cudaStream_t stream);
+369
View File
@@ -0,0 +1,369 @@
#include "Movienet.h"
#include <filesystem>
namespace ANSCENTER {
// Extract input size from _A<digit> suffix just before the file extension.
// _A0 / _A1 → 172 _A2 → 224 _A3 → 256 _A4 → 290 _A5 → 320
// Examples: Violence0305_A5.onnx → 320
// MyModel0908_A4.onnx → 290
// Returns 0 if the suffix is not recognised.
static int GetMovinetSizeFromFilename(const std::string& filename)
{
static constexpr int variantSize[] = {
172, // A0
172, // A1
224, // A2
256, // A3
290, // A4
320 // A5
};
// Strip extension — find last '.'
const size_t dotPos = filename.rfind('.');
if (dotPos == std::string::npos || dotPos < 3) return 0;
// Expect 3-char suffix just before the dot: _A<digit>
const char chUnderscore = filename[dotPos - 3];
const char chA = filename[dotPos - 2];
const char chDigit = filename[dotPos - 1];
if (chUnderscore == '_' &&
(chA == 'A' || chA == 'a') &&
chDigit >= '0' && chDigit <= '5')
{
return variantSize[chDigit - '0'];
}
return 0;
}
// Scan folder for any .onnx whose stem ends with _A0.._A5 (highest first).
// Returns the full path + matching input dimensions.
// If modelName is provided (e.g. "Violence0305_A2"), uses that exact model
// and resolves dimensions from its _A<digit> suffix.
// Falls back to "movinet.onnx" with 172×172 if nothing matches.
static std::string ResolveMovinetModel(const std::string& folder,
int& outWidth, int& outHeight,
const std::string& modelName = "")
{
// If a specific model name was given, use it directly
if (!modelName.empty()) {
std::string fname = modelName;
// Append .onnx if not already present
if (fname.size() < 5 || fname.substr(fname.size() - 5) != ".onnx") {
fname += ".onnx";
}
std::string fullPath = CreateFilePath(folder, fname);
int sz = GetMovinetSizeFromFilename(fname);
if (sz > 0) {
outWidth = sz;
outHeight = sz;
std::cout << "ANSMOVIENET: Using specified model '" << fname << "' with input size " << sz << "x" << sz << "\n";
}
else {
// Model name given but no recognized _A<digit> suffix — use default
outWidth = 172;
outHeight = 172;
std::cout << "ANSMOVIENET: Using specified model '" << fname << "' but failed to detect input size from filename. Defaulting to 172x172.\n";
}
return fullPath;
}
// No model name specified — scan folder, highest variant first
static const std::string suffixes[] = { "_A5", "_A4", "_A3", "_A2", "_A1", "_A0" };
try {
namespace fs = std::filesystem;
if (fs::is_directory(folder)) {
for (const auto& suffix : suffixes) {
for (const auto& entry : fs::directory_iterator(folder)) {
if (!entry.is_regular_file()) continue;
const std::string fname = entry.path().filename().string();
// Must be .onnx
if (fname.size() < 5) continue;
std::string ext = fname.substr(fname.size() - 5);
// Case-insensitive .onnx check
for (auto& c : ext) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
if (ext != ".onnx") continue;
// Check if stem ends with the current suffix
const std::string stem = fname.substr(0, fname.size() - 5);
if (stem.size() >= suffix.size() &&
stem.compare(stem.size() - suffix.size(), suffix.size(), suffix) == 0)
{
int sz = GetMovinetSizeFromFilename(fname);
if (sz > 0) {
outWidth = sz;
outHeight = sz;
return entry.path().string();
}
}
}
}
}
}
catch (...) {
// Filesystem error — fall through to legacy
std::cout << "ANSMOVIENET: Error scanning model folder '. Falling back to default model.\n";
}
// Legacy fallback
outWidth = 172;
outHeight = 172;
return CreateFilePath(folder, "movinet.onnx");
}
bool ANSMOVIENET::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
labelMap = "Face";
_licenseValid = true;
std::vector<std::string> labels{ labelMap };
if (!_licenseValid) return false;
try {
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::MOVIENET;
_modelConfig.detectionType = DetectionType::CLASSIFICATION;
// Auto-detect model variant and matching input size
int detectedW = 0, detectedH = 0;
std::string onnxModel = ResolveMovinetModel(_modelFolder, detectedW, detectedH);
_modelConfig.inpHeight = detectedH;
_modelConfig.inpWidth = detectedW;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_isInitialized) {
_movienet_detector.reset();
_isInitialized = false;
}
unsigned int numThreads = 1;
this->_movienet_detector = std::make_unique<MOVINET>(
onnxModel, TEMPORAL_LENGTH, detectedW, detectedH, 3, numThreads);
_isInitialized = true;
return true;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSMOVIENET::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSMOVIENET::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
int detectedW = 0, detectedH = 0;
std::string onnxModel = ResolveMovinetModel(_modelFolder, detectedW, detectedH);
_modelConfig.inpWidth = detectedW;
_modelConfig.inpHeight = detectedH;
unsigned int numThreads = 1;
_movienet_detector = std::make_unique<MOVINET>(
onnxModel, TEMPORAL_LENGTH, detectedW, detectedH, 3, numThreads);
_isInitialized = true;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSMOVIENET::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSMOVIENET::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) {
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap);
if (!result) return false;
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::MOVIENET;
_modelConfig.detectionType = DetectionType::CLASSIFICATION;
// Resolve model path and input dimensions.
// If modelName is given (e.g. "Violence0305_A2"), uses that exact model;
// otherwise scans the folder for the best _A<digit> variant.
int detectedW = 0, detectedH = 0;
std::string onnxModel = ResolveMovinetModel(modelFolder, detectedW, detectedH, modelName);
_modelConfig.inpWidth = detectedW;
_modelConfig.inpHeight = detectedH;
if (_modelConfig.modelMNSThreshold < 0.2)
_modelConfig.modelMNSThreshold = 0.5;
if (_modelConfig.modelConfThreshold < 0.2)
_modelConfig.modelConfThreshold = 0.5;
if (_isInitialized) {
_movienet_detector.reset();
_isInitialized = false;
}
unsigned int numThreads = 1;
this->_movienet_detector = std::make_unique<MOVINET>(
onnxModel, TEMPORAL_LENGTH, detectedW, detectedH, 3, numThreads);
_isInitialized = true;
return _isInitialized;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSMOVIENET::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSMOVIENET::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (FileExist(_modelFilePath)) {
optimizedModelFolder = GetParentFolder(_modelFilePath);
this->_logger.LogDebug("ANSMOVIENET::OptimizeModel", "This model is optimized. No need other optimization.", __FILE__, __LINE__);
return true;
}
else {
optimizedModelFolder = "";
this->_logger.LogFatal("ANSMOVIENET::OptimizeModel", "This model is not exist. Please check the model path again.", __FILE__, __LINE__);
return false;
}
}
ANSMOVIENET::~ANSMOVIENET() {
try {
Destroy();
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSMOVIENET::Destroy", e.what(), __FILE__, __LINE__);
}
}
bool ANSMOVIENET::Destroy() {
try {
std::lock_guard<std::recursive_mutex> lock(_mutex);
_cameraQueues.clear();
_globalFrameCounter = 0;
_movienet_detector.reset();
_isInitialized = false;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSMOVIENET::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
// Inference functions
std::vector<Object> ANSMOVIENET::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::vector<Object> result = Inference(input, camera_id);
if (_trackerEnabled) {
result = ApplyTracking(result, camera_id);
if (_stabilizationEnabled) result = StabilizeDetections(result, camera_id);
}
return result;
}
std::vector<Object> ANSMOVIENET::RunInference(const cv::Mat& input) {
return Inference(input, "MovienetCam");
}
void ANSMOVIENET::CleanupStaleQueues() {
// Called internally � already under lock from Inference()
if (_cameraQueues.empty()) return;
// 1. Remove queues not accessed for STALE_THRESHOLD frames
for (auto it = _cameraQueues.begin(); it != _cameraQueues.end(); ) {
int age = _globalFrameCounter - it->second.lastAccessFrame;
if (age > STALE_THRESHOLD) {
it = _cameraQueues.erase(it);
}
else {
++it;
}
}
// 2. If still over hard cap, remove oldest queues first
if (static_cast<int>(_cameraQueues.size()) > MAX_QUEUES) {
// Collect and sort by last access time
std::vector<std::pair<int, std::string>> accessTimes;
accessTimes.reserve(_cameraQueues.size());
for (const auto& [key, state] : _cameraQueues) {
accessTimes.emplace_back(state.lastAccessFrame, key);
}
std::sort(accessTimes.begin(), accessTimes.end());
// Remove oldest until under cap
int toRemove = static_cast<int>(_cameraQueues.size()) - MAX_QUEUES;
for (int i = 0; i < toRemove && i < static_cast<int>(accessTimes.size()); ++i) {
_cameraQueues.erase(accessTimes[i].second);
}
}
}
std::vector<Object> ANSMOVIENET::Inference(const cv::Mat& input, const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
std::vector<Object> detectedObjects;
if (!_isInitialized || !_licenseValid || !_movienet_detector) {
this->_logger.LogError("ANSMOVIENET::Inference",
"Model is not initialized or license is not valid", __FILE__, __LINE__);
return detectedObjects;
}
if (input.empty()) {
this->_logger.LogError("ANSMOVIENET::Inference",
"Input frame is empty", __FILE__, __LINE__);
return detectedObjects;
}
try {
_globalFrameCounter++;
std::string cameraKey = camera_id.empty() ? "" : camera_id;
auto& state = _cameraQueues[cameraKey];
// Update access timestamp
state.lastAccessFrame = _globalFrameCounter;
state.frameCount++;
// Add frame to queue
state.frames.push_back(input.clone());
// Maintain queue at TEMPORAL_LENGTH
while (state.frames.size() > static_cast<size_t>(TEMPORAL_LENGTH)) {
state.frames.pop_front();
}
// Run inference when:
// 1. Full window available
// 2. Stride condition met
bool hasFullWindow = (state.frames.size() == static_cast<size_t>(TEMPORAL_LENGTH));
bool strideReady = (state.frameCount == TEMPORAL_LENGTH)
|| (state.frameCount > TEMPORAL_LENGTH
&& (state.frameCount - TEMPORAL_LENGTH) % _inferenceStride == 0);
if (hasFullWindow && strideReady) {
std::pair<int, float> result;
_movienet_detector->inference(state.frames, result);
if (result.first >= 0) {
Object obj;
obj.classId = result.first;
obj.className = (result.first < static_cast<int>(_classes.size())) ?
_classes[result.first] : "Unknown";
obj.confidence = result.second;
obj.box = cv::Rect(0, 0, input.cols, input.rows);
detectedObjects.push_back(obj);
}
}
// Store in base class camera data
CameraData& cameraData = GetCameraData(cameraKey);
cameraData._detectionQueue.push_back(detectedObjects);
if (cameraData._detectionQueue.size() > QUEUE_SIZE) {
cameraData._detectionQueue.pop_front();
}
// ----- Periodic self-cleanup -----
if (_globalFrameCounter % CLEANUP_INTERVAL == 0) {
CleanupStaleQueues();
}
}
catch (const std::exception& e) {
this->_logger.LogError("ANSMOVIENET::Inference",
std::string("Exception during inference: ") + e.what(), __FILE__, __LINE__);
detectedObjects.clear();
}
catch (...) {
this->_logger.LogError("ANSMOVIENET::Inference",
"Unknown exception during inference", __FILE__, __LINE__);
detectedObjects.clear();
}
return detectedObjects;
}
}
+53
View File
@@ -0,0 +1,53 @@
#ifndef MOVENET_H
#define MOVENET_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
#include "engine.h"
#include "ONNXEngine.h"
//#define FACEDEBUG
namespace ANSCENTER {
class ANSENGINE_API ANSMOVIENET :public ANSODBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSMOVIENET();
private:
std::string _modelFilePath;
std::unique_ptr<MOVINET> _movienet_detector = nullptr;
static constexpr int TEMPORAL_LENGTH = 16;
// ----- Per-camera queue state -----
struct CameraQueueState {
std::deque<cv::Mat> frames;
int frameCount = 0; // total frames received (for stride calc)
int lastAccessFrame = 0; // global frame when last accessed (for staleness)
};
std::unordered_map<std::string, CameraQueueState> _cameraQueues;
// ----- Inference stride -----
int _inferenceStride = TEMPORAL_LENGTH; // non-overlapping by default
// ----- Global frame counter for staleness tracking -----
int _globalFrameCounter = 0;
// ----- Cleanup config -----
static constexpr int STALE_THRESHOLD = 100; // frames without access → stale
static constexpr int CLEANUP_INTERVAL = 50; // run cleanup every N frames
static constexpr int MAX_QUEUES = 200; // hard cap on total queues
// ----- Internal methods -----
std::vector<Object> Inference(const cv::Mat& input, const std::string& camera_id);
void CleanupStaleQueues();
};
}
#endif
@@ -0,0 +1,970 @@
#include "NV12PreprocessHelper.h"
#include "ANSGpuFrameRegistry.h"
#include "ANSEngineCommon.h"
#include <opencv2/cudaimgproc.hpp>
#include <opencv2/cudawarping.hpp>
#include <opencv2/core/cuda_stream_accessor.hpp>
#include <cstring>
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
// SEH-protected memcpy: returns false on access violation instead of crashing.
// Used for NV12 plane data which may originate from D3D11VA staging textures.
static bool safeMemcpy(void* dst, const void* src, size_t bytes) {
__try {
std::memcpy(dst, src, bytes);
return true;
}
__except (EXCEPTION_EXECUTE_HANDLER) {
return false;
}
}
// Check if a memory range is readable (non-crashing probe via VirtualQuery)
static bool isMemoryReadable(const void* ptr, size_t bytes) {
if (!ptr || bytes == 0) return false;
MEMORY_BASIC_INFORMATION mbi{};
if (VirtualQuery(ptr, &mbi, sizeof(mbi)) == 0) return false;
if (mbi.State != MEM_COMMIT) return false;
if (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) return false;
const DWORD readFlags = PAGE_READONLY | PAGE_READWRITE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE;
return (mbi.Protect & readFlags) != 0;
}
#else
static bool safeMemcpy(void* dst, const void* src, size_t bytes) {
std::memcpy(dst, src, bytes);
return true;
}
static bool isMemoryReadable(const void*, size_t) { return true; }
#endif
// NV12→RGB CUDA kernel (nv12_to_rgb.cu)
extern "C" void launchNV12ToRGB(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int width, int height,
cudaStream_t stream);
extern "C" void launchNV12ToRGBLetterbox(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
int unpadW, int unpadH,
float invScale,
cudaStream_t stream);
extern "C" void launchNV12ToRGBCenterLetterbox(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
int unpadW, int unpadH,
float invScale,
int padLeft, int padTop,
cudaStream_t stream);
extern "C" void launchNV12AffineWarpToBGR(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
float m00, float m01, float m02,
float m10, float m11, float m12,
cudaStream_t stream);
extern "C" void launchNV12ToRGBResize(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream);
extern "C" void launchNV12ToRGBResizeCHW_uint8(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* chwOut,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream);
extern "C" void launchNV12ToRGBResizeCHW_float(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
float* chwOut,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream);
extern "C" void launchNV12ToBGRResize(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream);
extern "C" void launchNV12CropToBGR(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH,
int cropX, int cropY,
int srcW, int srcH,
cudaStream_t stream);
namespace ANSCENTER {
NV12PreprocessHelper::~NV12PreprocessHelper() {
destroy();
}
void NV12PreprocessHelper::destroy() {
if (m_pinnedBuf) {
cudaFreeHost(m_pinnedBuf);
m_pinnedBuf = nullptr;
m_pinnedBufSize = 0;
}
}
void NV12PreprocessHelper::ensurePinnedBuffer(size_t bytes, SPDLogger& logger, const char* tag) {
if (bytes <= m_pinnedBufSize) return;
if (m_pinnedBuf) {
cudaFreeHost(m_pinnedBuf);
m_pinnedBuf = nullptr;
m_pinnedBufSize = 0;
}
cudaError_t err = cudaHostAlloc(&m_pinnedBuf, bytes, cudaHostAllocPortable);
if (err != cudaSuccess) {
if (!m_nv12PinnedLogged) {
m_nv12PinnedLogged = true;
logger.LogWarn(std::string(tag) + "::NV12Helper",
"cudaHostAlloc failed (" + std::string(cudaGetErrorString(err)) +
") for " + std::to_string(bytes) + " bytes",
__FILE__, __LINE__);
}
m_pinnedBuf = nullptr;
m_pinnedBufSize = 0;
return;
}
m_pinnedBufSize = bytes;
}
bool NV12PreprocessHelper::isCudaContextHealthy(SPDLogger& logger, const char* tag) {
if (m_cudaContextDead) return false;
cudaError_t pending = cudaGetLastError();
cudaError_t err = cudaStreamQuery(nullptr);
if (err == cudaSuccess || err == cudaErrorNotReady) {
m_cudaFailStreak = 0;
return true;
}
++m_cudaFailStreak;
if (m_cudaFailStreak >= CUDA_FAIL_LIMIT && !m_cudaContextDead) {
m_cudaContextDead = true;
logger.LogFatal(std::string(tag) + "::NV12Helper",
"CUDA context permanently corrupted after " +
std::to_string(m_cudaFailStreak) +
" consecutive failures (last error: " +
std::string(cudaGetErrorString(err)) +
", pending: " + std::string(cudaGetErrorString(pending)) +
"). GPU preprocessing DISABLED for this engine instance. "
"This is typically caused by NVDEC/CUVID surface pool exhaustion "
"(Error mapping a picture with CUVID: 205). "
"Reduce the number of concurrent HW-decoded 4K streams or "
"restart the application to recover.",
__FILE__, __LINE__);
}
return false;
}
// ====================================================================
// tryNV12 — attempt NV12 fast-path preprocessing
// ====================================================================
NV12Result NV12PreprocessHelper::tryNV12(
const cv::Mat& inputImage, int inferenceGpu,
int inputW, int inputH,
const NV12KernelLauncher& launcher,
SPDLogger& logger, const char* tag) {
NV12Result result;
// Skip during warmup phase
if (m_inferenceCount < NV12_WARMUP_THRESHOLD) {
return result;
}
// Get NV12 frame data from thread-local (set by RunInferenceComplete_LV)
GpuFrameData* gpuData = tl_currentGpuFrame();
if (!gpuData || !gpuData->yPlane) {
return result; // no NV12 data — caller uses BGR
}
// ── BGR full-res path (pixFmt=1000 from ANSVideoPlayer) ──────────
if (gpuData->pixelFormat == 1000) {
cv::Mat fullResImg(gpuData->height, gpuData->width, CV_8UC3,
gpuData->yPlane, gpuData->yLinesize);
if (!m_bgrFullResLogged) {
m_bgrFullResLogged = true;
logger.LogInfo(std::string(tag) + "::NV12Helper",
"BGR full-res ACTIVE: " + std::to_string(gpuData->width) + "x" +
std::to_string(gpuData->height) + " -> inference (display=" +
std::to_string(inputImage.cols) + "x" + std::to_string(inputImage.rows) + ")",
__FILE__, __LINE__);
}
result.useBgrFullRes = true;
result.bgrFullResImg = fullResImg.clone(); // clone under lock
if (inputImage.cols > 0 && inputImage.rows > 0 &&
(fullResImg.cols != inputImage.cols || fullResImg.rows != inputImage.rows)) {
result.bgrFullResScaleX = static_cast<float>(inputImage.cols) / static_cast<float>(fullResImg.cols);
result.bgrFullResScaleY = static_cast<float>(inputImage.rows) / static_cast<float>(fullResImg.rows);
}
// (No registry lock — data kept alive by refcount)
return result;
}
// ── NV12 path (pixFmt=23 = AV_PIX_FMT_NV12) ────────────────────
if (gpuData->pixelFormat != 23) {
return result; // unknown format — fall back to BGR
}
if (!gpuData->uvPlane) {
if (!m_nv12NullLogged) {
m_nv12NullLogged = true;
logger.LogWarn(std::string(tag) + "::NV12Helper",
"Early exit: null uvPlane",
__FILE__, __LINE__);
}
return result;
}
const int frameW = gpuData->width;
const int frameH = gpuData->height;
if (frameW <= 0 || frameH <= 0) {
if (!m_nv12DimLogged) {
m_nv12DimLogged = true;
logger.LogWarn(std::string(tag) + "::NV12Helper",
"Early exit: bad dimensions — w=" + std::to_string(frameW) +
" h=" + std::to_string(frameH),
__FILE__, __LINE__);
}
return result;
}
if (m_cudaContextDead) {
if (!m_nv12DeadLogged) {
m_nv12DeadLogged = true;
logger.LogWarn(std::string(tag) + "::NV12Helper",
"Early exit: CUDA context dead",
__FILE__, __LINE__);
}
return result;
}
const bool isCudaDevice = gpuData->isCudaDevicePtr;
const bool gpuMatch = !isCudaDevice ||
gpuData->gpuIndex < 0 ||
gpuData->gpuIndex == inferenceGpu;
const bool useZeroCopy = isCudaDevice && gpuMatch;
// Effective plane pointers — for zero-copy, use CUDA device ptrs;
// for CPU upload, use the CPU snapshot buffers.
uint8_t* effYPlane;
uint8_t* effUvPlane;
int effYLinesize;
int effUvLinesize;
if (useZeroCopy) {
// Same GPU: wrap NVDEC device pointers directly
effYPlane = gpuData->yPlane;
effUvPlane = gpuData->uvPlane;
effYLinesize = gpuData->yLinesize;
effUvLinesize = gpuData->uvLinesize;
} else if (isCudaDevice && !gpuMatch) {
// Cross-GPU: can't use CUDA device ptrs from another GPU context.
// Fall through to CPU upload using the CPU snapshot buffers.
if (gpuData->cpuYPlane && gpuData->cpuUvPlane) {
effYPlane = gpuData->cpuYPlane;
effUvPlane = gpuData->cpuUvPlane;
effYLinesize = gpuData->cpuYLinesize;
effUvLinesize = gpuData->cpuUvLinesize;
if (!m_gpuMismatchLogged) {
m_gpuMismatchLogged = true;
logger.LogInfo(std::string(tag) + "::NV12Helper",
"GPU mismatch (decode GPU " + std::to_string(gpuData->gpuIndex) +
" vs inference GPU " + std::to_string(inferenceGpu) +
") — using CPU NV12 upload fallback",
__FILE__, __LINE__);
}
} else {
// No CPU fallback available — fall back to BGR
if (!m_gpuMismatchLogged) {
m_gpuMismatchLogged = true;
logger.LogInfo(std::string(tag) + "::NV12Helper",
"GPU mismatch (decode GPU " + std::to_string(gpuData->gpuIndex) +
" vs inference GPU " + std::to_string(inferenceGpu) +
") — no CPU fallback, using BGR",
__FILE__, __LINE__);
}
return result;
}
} else {
// Not CUDA device ptrs: use whatever yPlane/uvPlane point to (CPU data)
effYPlane = gpuData->yPlane;
effUvPlane = gpuData->uvPlane;
effYLinesize = gpuData->yLinesize;
effUvLinesize = gpuData->uvLinesize;
}
// Log path selection once
if (!m_nv12PathLogged) {
m_nv12PathLogged = true;
const char* pathName = useZeroCopy ? "CUDA_ZERO_COPY" : "CPU_NV12_UPLOAD";
logger.LogInfo(std::string(tag) + "::NV12Helper",
std::string("Path: ") + pathName +
" | isCuda=" + std::to_string(isCudaDevice) +
" gpuMatch=" + std::to_string(gpuMatch) +
" decodeGpu=" + std::to_string(gpuData->gpuIndex) +
" infGpu=" + std::to_string(inferenceGpu) +
" frame=" + std::to_string(frameW) + "x" + std::to_string(frameH),
__FILE__, __LINE__);
}
cv::cuda::Stream stream;
cv::cuda::GpuMat gpuY, gpuUV;
if (useZeroCopy) {
// CUDA zero-copy: wrap NVDEC device pointers directly
gpuY = cv::cuda::GpuMat(frameH, frameW, CV_8UC1,
effYPlane, static_cast<size_t>(effYLinesize));
gpuUV = cv::cuda::GpuMat(frameH / 2, frameW, CV_8UC1,
effUvPlane, static_cast<size_t>(effUvLinesize));
} else {
// CPU path: memcpy + upload
const size_t ySize = static_cast<size_t>(frameW) * frameH;
const size_t uvSize = static_cast<size_t>(frameW) * frameH / 2;
const size_t nv12Size = ySize + uvSize;
ensurePinnedBuffer(nv12Size, logger, tag);
if (!m_pinnedBuf) {
return result;
}
// Validate NV12 plane pointers before memcpy
const size_t yBufNeeded = (effYLinesize == frameW)
? ySize
: static_cast<size_t>(effYLinesize) * frameH;
const size_t uvBufNeeded = (effUvLinesize == frameW)
? uvSize
: static_cast<size_t>(effUvLinesize) * (frameH / 2);
if (!isMemoryReadable(effYPlane, std::min(yBufNeeded, (size_t)4096)) ||
!isMemoryReadable(effUvPlane, std::min(uvBufNeeded, (size_t)4096))) {
logger.LogWarn(std::string(tag) + "::NV12Helper",
"NV12 plane pointers not readable — falling back to BGR",
__FILE__, __LINE__);
return result;
}
uint8_t* dst = static_cast<uint8_t*>(m_pinnedBuf);
bool cpyOk = true;
if (effYLinesize == frameW) {
cpyOk = safeMemcpy(dst, effYPlane, ySize);
} else {
for (int row = 0; row < frameH && cpyOk; row++)
cpyOk = safeMemcpy(dst + row * frameW,
effYPlane + row * effYLinesize, frameW);
}
if (cpyOk) {
uint8_t* uvDst = dst + ySize;
if (effUvLinesize == frameW) {
cpyOk = safeMemcpy(uvDst, effUvPlane, uvSize);
} else {
for (int row = 0; row < frameH / 2 && cpyOk; row++)
cpyOk = safeMemcpy(uvDst + row * frameW,
effUvPlane + row * effUvLinesize, frameW);
}
}
if (!cpyOk) {
logger.LogWarn(std::string(tag) + "::NV12Helper",
"Access violation during NV12 memcpy — falling back to BGR",
__FILE__, __LINE__);
return result;
}
// NV12 data safely in pinned memory — release registry lock
// (No registry lock to release — data kept alive by refcount)
cv::Mat pinnedY(frameH, frameW, CV_8UC1, m_pinnedBuf);
cv::Mat pinnedUV(frameH / 2, frameW, CV_8UC1,
static_cast<uint8_t*>(m_pinnedBuf) + ySize);
gpuY.upload(pinnedY, stream);
gpuUV.upload(pinnedUV, stream);
}
// Use display dimensions for coordinate mapping
const float metaW = static_cast<float>(inputImage.cols > 0 ? inputImage.cols : frameW);
const float metaH = static_cast<float>(inputImage.rows > 0 ? inputImage.rows : frameH);
if (metaH <= 0 || metaW <= 0) {
if (!m_nv12MetaLogged) {
m_nv12MetaLogged = true;
logger.LogWarn(std::string(tag) + "::NV12Helper",
"Early exit: metadata dims invalid",
__FILE__, __LINE__);
}
return result;
}
result.metaWidth = metaW;
result.metaHeight = metaH;
result.ratio = 1.f / std::min(
static_cast<float>(inputW) / metaW,
static_cast<float>(inputH) / metaH);
// Launch engine-specific kernel
cv::cuda::GpuMat gpuResized;
gpuResized.create(inputH, inputW, CV_8UC3);
cudaStream_t rawStream = cv::cuda::StreamAccessor::getStream(stream);
launcher(gpuY, gpuUV, gpuResized, frameW, frameH, inputW, inputH, rawStream);
stream.waitForCompletion();
// (No registry lock to release — data kept alive by refcount)
// Log NV12 fast-path usage once per instance
if (!m_nv12ActiveLogged) {
m_nv12ActiveLogged = true;
const char* mode = useZeroCopy ? "CUDA zero-copy" : "CPU upload";
logger.LogInfo(std::string(tag) + "::NV12Helper",
std::string(mode) + " ACTIVE: " +
std::to_string(frameW) + "x" + std::to_string(frameH) +
" NV12 -> " + std::to_string(inputW) + "x" + std::to_string(inputH) +
" display=" + std::to_string(static_cast<int>(metaW)) + "x" +
std::to_string(static_cast<int>(metaH)),
__FILE__, __LINE__);
}
result.succeeded = true;
result.gpuRGB = std::move(gpuResized);
return result;
}
// ====================================================================
// Default YOLO kernel launcher
//
// For detection/seg/pose/OBB: fused NV12→RGB + letterbox in a single
// kernel at model input resolution (e.g. 640x640).
// For classification: separate NV12→RGB at full res, then cuda::resize.
// (Classification detection is not done here — the caller's engine
// should set up the appropriate launcher if needed. This default
// always uses the fused letterbox path, which works for all YOLO
// detection/seg/pose/OBB tasks.)
// ====================================================================
NV12KernelLauncher NV12PreprocessHelper::defaultYOLOLauncher() {
return [](const cv::cuda::GpuMat& gpuY, const cv::cuda::GpuMat& gpuUV,
cv::cuda::GpuMat& gpuOut, int srcW, int srcH,
int inputW, int inputH, cudaStream_t stream) {
if (srcW == inputW && srcH == inputH) {
// Source matches model input — direct NV12→RGB, no resize needed
launchNV12ToRGB(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuOut.ptr<uint8_t>(), static_cast<int>(gpuOut.step),
srcW, srcH, stream);
} else {
// Fused NV12→RGB + letterbox resize
float r = std::min(static_cast<float>(inputW) / srcW,
static_cast<float>(inputH) / srcH);
int unpadW = static_cast<int>(r * srcW);
int unpadH = static_cast<int>(r * srcH);
float invScale = 1.0f / r;
launchNV12ToRGBLetterbox(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuOut.ptr<uint8_t>(), static_cast<int>(gpuOut.step),
inputW, inputH,
srcW, srcH,
unpadW, unpadH,
invScale, stream);
}
};
}
// ====================================================================
// Classification launcher (direct resize, no letterbox)
// ====================================================================
NV12KernelLauncher NV12PreprocessHelper::classificationLauncher() {
return [](const cv::cuda::GpuMat& gpuY, const cv::cuda::GpuMat& gpuUV,
cv::cuda::GpuMat& gpuOut, int srcW, int srcH,
int inputW, int inputH, cudaStream_t stream) {
if (srcW == inputW && srcH == inputH) {
// Source matches model input — direct NV12→RGB, no resize needed
launchNV12ToRGB(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuOut.ptr<uint8_t>(), static_cast<int>(gpuOut.step),
srcW, srcH, stream);
} else {
// Fused NV12→RGB + direct resize (stretch to fill, no padding)
launchNV12ToRGBResize(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuOut.ptr<uint8_t>(), static_cast<int>(gpuOut.step),
inputW, inputH,
srcW, srcH, stream);
}
};
}
// ====================================================================
// NV12→BGR fused resize (for OCR detection)
// ====================================================================
void NV12PreprocessHelper::nv12ToBGRResize(
const uint8_t* devY, int yPitch,
const uint8_t* devUV, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH, int srcW, int srcH,
cudaStream_t stream) {
launchNV12ToBGRResize(devY, yPitch, devUV, uvPitch,
bgrOut, outPitch, outW, outH, srcW, srcH, stream);
}
// SCRFD center-padded letterbox launcher
// ====================================================================
NV12KernelLauncher NV12PreprocessHelper::scrfdCenterLetterboxLauncher(int padLeft, int padTop) {
return [padLeft, padTop](const cv::cuda::GpuMat& gpuY, const cv::cuda::GpuMat& gpuUV,
cv::cuda::GpuMat& gpuOut, int srcW, int srcH,
int inputW, int inputH, cudaStream_t stream) {
float r = std::min(static_cast<float>(inputW) / srcW,
static_cast<float>(inputH) / srcH);
int unpadW = static_cast<int>(r * srcW);
int unpadH = static_cast<int>(r * srcH);
float invScale = 1.0f / r;
launchNV12ToRGBCenterLetterbox(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuOut.ptr<uint8_t>(), static_cast<int>(gpuOut.step),
inputW, inputH,
srcW, srcH,
unpadW, unpadH,
invScale,
padLeft, padTop,
stream);
};
}
// ====================================================================
// NV12 affine warp for face alignment
//
// Reads full-res NV12 from the registry, applies inverse affine
// transform via CUDA kernel, outputs a small aligned face in BGR.
// The affine matrix is computed on CPU (from detected landmarks to
// ArcFace template), then scaled to map to full-res NV12 source.
// ====================================================================
NV12PreprocessHelper::NV12AffineResult NV12PreprocessHelper::tryNV12AffineWarp(
const cv::Mat& inputImage, int inferenceGpu,
const cv::Mat& affineMatrix, int outW, int outH,
float scaleX, float scaleY,
SPDLogger& logger, const char* tag) {
NV12AffineResult result;
// Skip during warmup
if (m_inferenceCount < NV12_WARMUP_THRESHOLD) {
return result;
}
if (affineMatrix.empty()) {
return result;
}
// Get NV12 frame data from thread-local (set by RunInferenceComplete_LV)
GpuFrameData* gpuData = tl_currentGpuFrame();
if (!gpuData || !gpuData->yPlane || !gpuData->uvPlane || gpuData->pixelFormat != 23) {
return result;
}
const int frameW = gpuData->width;
const int frameH = gpuData->height;
if (frameW <= 0 || frameH <= 0 || m_cudaContextDead) {
return result;
}
const bool isCudaDevice = gpuData->isCudaDevicePtr;
const bool gpuMatch = !isCudaDevice ||
gpuData->gpuIndex < 0 ||
gpuData->gpuIndex == inferenceGpu;
if (isCudaDevice && !gpuMatch) {
return result; // cross-GPU — fall back to CPU warpAffine
}
const bool useZeroCopy = isCudaDevice && gpuMatch;
uint8_t* effYPlane = gpuData->yPlane;
uint8_t* effUvPlane = gpuData->uvPlane;
int effYLinesize = gpuData->yLinesize;
int effUvLinesize = gpuData->uvLinesize;
// Compute inverse affine matrix and scale for full-res NV12
cv::Mat M_inv;
cv::invertAffineTransform(affineMatrix, M_inv);
M_inv.convertTo(M_inv, CV_64F);
// Scale inverse affine to map from 112x112 output → full-res NV12 source
// Row 0 maps to X (multiply by scaleX), Row 1 maps to Y (multiply by scaleY)
double* row0 = M_inv.ptr<double>(0);
double* row1 = M_inv.ptr<double>(1);
row0[0] *= scaleX; row0[1] *= scaleX; row0[2] *= scaleX;
row1[0] *= scaleY; row1[1] *= scaleY; row1[2] *= scaleY;
float m00 = static_cast<float>(row0[0]);
float m01 = static_cast<float>(row0[1]);
float m02 = static_cast<float>(row0[2]);
float m10 = static_cast<float>(row1[0]);
float m11 = static_cast<float>(row1[1]);
float m12 = static_cast<float>(row1[2]);
cv::cuda::Stream stream;
cv::cuda::GpuMat gpuY, gpuUV;
if (useZeroCopy) {
gpuY = cv::cuda::GpuMat(frameH, frameW, CV_8UC1,
effYPlane, static_cast<size_t>(effYLinesize));
gpuUV = cv::cuda::GpuMat(frameH / 2, frameW, CV_8UC1,
effUvPlane, static_cast<size_t>(effUvLinesize));
} else {
// CPU path: memcpy + upload
const size_t ySize = static_cast<size_t>(frameW) * frameH;
const size_t uvSize = static_cast<size_t>(frameW) * frameH / 2;
const size_t nv12Size = ySize + uvSize;
ensurePinnedBuffer(nv12Size, logger, tag);
if (!m_pinnedBuf) {
return result;
}
uint8_t* dst = static_cast<uint8_t*>(m_pinnedBuf);
bool cpyOk = true;
if (effYLinesize == frameW) {
cpyOk = safeMemcpy(dst, effYPlane, ySize);
} else {
for (int row = 0; row < frameH && cpyOk; row++)
cpyOk = safeMemcpy(dst + row * frameW,
effYPlane + row * effYLinesize, frameW);
}
if (cpyOk) {
uint8_t* uvDst = dst + ySize;
if (effUvLinesize == frameW) {
cpyOk = safeMemcpy(uvDst, effUvPlane, uvSize);
} else {
for (int row = 0; row < frameH / 2 && cpyOk; row++)
cpyOk = safeMemcpy(uvDst + row * frameW,
effUvPlane + row * effUvLinesize, frameW);
}
}
if (!cpyOk) {
return result;
}
// (No registry lock to release — data kept alive by refcount)
cv::Mat pinnedY(frameH, frameW, CV_8UC1, m_pinnedBuf);
cv::Mat pinnedUV(frameH / 2, frameW, CV_8UC1,
static_cast<uint8_t*>(m_pinnedBuf) + ySize);
gpuY.upload(pinnedY, stream);
gpuUV.upload(pinnedUV, stream);
}
// Launch affine warp kernel (outputs BGR 112x112)
cv::cuda::GpuMat gpuFace(outH, outW, CV_8UC3);
cudaStream_t rawStream = cv::cuda::StreamAccessor::getStream(stream);
launchNV12AffineWarpToBGR(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuFace.ptr<uint8_t>(), static_cast<int>(gpuFace.step),
outW, outH,
frameW, frameH,
m00, m01, m02, m10, m11, m12,
rawStream);
stream.waitForCompletion();
// (No registry lock to release — data kept alive by refcount)
// Keep GPU copy for recognizer (avoids re-upload), download CPU copy for mask
result.gpuAlignedFace = gpuFace;
gpuFace.download(result.alignedFaceBGR);
result.succeeded = true;
return result;
}
// ── NV12 rectangular crop → BGR ────────────────────────────────────
NV12PreprocessHelper::NV12CropResult NV12PreprocessHelper::tryNV12CropToBGR(
const cv::Mat& inputImage, int inferenceGpu,
const cv::Rect& bbox, int padding,
float scaleX, float scaleY,
SPDLogger& logger, const char* tag)
{
NV12CropResult result;
// Warmup gating
if (m_inferenceCount < NV12_WARMUP_THRESHOLD)
return result;
// CUDA health check
if (m_cudaContextDead)
return result;
// Get NV12 frame data from thread-local (set by RunInferenceComplete_LV)
auto* gpuData = tl_currentGpuFrame();
if (!gpuData || gpuData->pixelFormat != 23)
return result; // no NV12 data
if (!gpuData->isCudaDevicePtr || !gpuData->yPlane || !gpuData->uvPlane)
return result; // NV12 not on GPU
const int frameW = gpuData->width;
const int frameH = gpuData->height;
// Scale bbox from display-res to full-res NV12 coords with padding
const int fx1 = std::max(0, static_cast<int>((bbox.x - padding) * scaleX));
const int fy1 = std::max(0, static_cast<int>((bbox.y - padding) * scaleY));
const int fx2 = std::min(frameW, static_cast<int>((bbox.x + bbox.width + padding) * scaleX));
const int fy2 = std::min(frameH, static_cast<int>((bbox.y + bbox.height + padding) * scaleY));
const int cropW = fx2 - fx1;
const int cropH = fy2 - fy1;
if (cropW < 5 || cropH < 5)
return result;
// GPU device matching
int currentDev = 0;
cudaGetDevice(&currentDev);
if (inferenceGpu >= 0 && currentDev != inferenceGpu)
cudaSetDevice(inferenceGpu);
// Wrap existing GPU NV12 planes
cv::cuda::GpuMat gpuY(frameH, frameW, CV_8UC1,
gpuData->yPlane, gpuData->yLinesize);
cv::cuda::GpuMat gpuUV(frameH / 2, frameW / 2, CV_8UC2,
gpuData->uvPlane, gpuData->uvLinesize);
// Allocate output on GPU
cv::cuda::GpuMat gpuCrop(cropH, cropW, CV_8UC3);
// Launch crop kernel
cv::cuda::Stream stream;
cudaStream_t rawStream = cv::cuda::StreamAccessor::getStream(stream);
launchNV12CropToBGR(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
gpuCrop.ptr<uint8_t>(), static_cast<int>(gpuCrop.step),
cropW, cropH,
fx1, fy1,
frameW, frameH,
rawStream);
stream.waitForCompletion();
// (No registry lock to release — data kept alive by refcount)
// Download small crop to CPU
gpuCrop.download(result.bgrCrop);
result.succeeded = true;
// One-shot diagnostic log
if (!m_nv12CropLogged) {
m_nv12CropLogged = true;
logger.LogInfo(tag,
"NV12 GPU crop ACTIVE: " + std::to_string(cropW) + "x" + std::to_string(cropH) +
" BGR cropped from " + std::to_string(frameW) + "x" + std::to_string(frameH) +
" NV12 (display=" + std::to_string(inputImage.cols) + "x" + std::to_string(inputImage.rows) +
" scaleX=" + std::to_string(scaleX) + " scaleY=" + std::to_string(scaleY) + ")",
__FILE__, __LINE__);
}
return result;
}
// ====================================================================
// tryNV12DirectToBuffer — write NV12→RGB CHW directly into TRT buffer
//
// For engines that manage their own GPU buffers (e.g. SAM3 with raw
// TRT enqueueV3). Same registry lookup / GPU matching / safety logic
// as tryNV12(), but writes CHW planar output into a pre-allocated
// void* GPU buffer instead of allocating an HWC GpuMat.
// ====================================================================
NV12PreprocessHelper::NV12DirectResult NV12PreprocessHelper::tryNV12DirectToBuffer(
const cv::Mat& inputImage, int inferenceGpu,
void* dstGpuBuffer, int inputW, int inputH,
bool isFloat32,
cudaStream_t stream,
SPDLogger& logger, const char* tag) {
NV12DirectResult result;
// Skip during warmup phase
if (m_inferenceCount < NV12_WARMUP_THRESHOLD) {
return result;
}
if (!dstGpuBuffer) return result;
// Get NV12 frame data from thread-local (set by RunInferenceComplete_LV)
GpuFrameData* gpuData = tl_currentGpuFrame();
if (!gpuData || !gpuData->yPlane) {
return result; // no NV12 data — caller uses CPU path
}
// BGR full-res path not applicable for direct-to-buffer (SAM3 handles its own resize)
if (gpuData->pixelFormat == 1000) {
return result;
}
// NV12 path only
if (gpuData->pixelFormat != 23) {
return result;
}
if (!gpuData->uvPlane) {
return result;
}
const int frameW = gpuData->width;
const int frameH = gpuData->height;
if (frameW <= 0 || frameH <= 0 || m_cudaContextDead) {
return result;
}
const bool isCudaDevice = gpuData->isCudaDevicePtr;
const bool gpuMatch = !isCudaDevice ||
gpuData->gpuIndex < 0 ||
gpuData->gpuIndex == inferenceGpu;
const bool useZeroCopy = isCudaDevice && gpuMatch;
if (isCudaDevice && !gpuMatch) {
return result; // cross-GPU — fall back to CPU path
}
uint8_t* effYPlane = gpuData->yPlane;
uint8_t* effUvPlane = gpuData->uvPlane;
int effYLinesize = gpuData->yLinesize;
int effUvLinesize = gpuData->uvLinesize;
// Log path selection once
if (!m_nv12PathLogged) {
m_nv12PathLogged = true;
const char* pathName = useZeroCopy ? "CUDA_ZERO_COPY" : "CPU_NV12_UPLOAD";
logger.LogInfo(std::string(tag) + "::NV12Helper",
std::string("Path: ") + pathName +
" | isCuda=" + std::to_string(isCudaDevice) +
" gpuMatch=" + std::to_string(gpuMatch) +
" frame=" + std::to_string(frameW) + "x" + std::to_string(frameH) +
" (direct CHW " + (isFloat32 ? "float32" : "uint8") + ")",
__FILE__, __LINE__);
}
cv::cuda::GpuMat gpuY, gpuUV;
if (useZeroCopy) {
gpuY = cv::cuda::GpuMat(frameH, frameW, CV_8UC1,
effYPlane, static_cast<size_t>(effYLinesize));
gpuUV = cv::cuda::GpuMat(frameH / 2, frameW, CV_8UC1,
effUvPlane, static_cast<size_t>(effUvLinesize));
} else {
// CPU path: memcpy NV12 to pinned buffer, upload
const size_t ySize = static_cast<size_t>(frameW) * frameH;
const size_t uvSize = static_cast<size_t>(frameW) * frameH / 2;
const size_t nv12Size = ySize + uvSize;
ensurePinnedBuffer(nv12Size, logger, tag);
if (!m_pinnedBuf) return result;
if (!isMemoryReadable(effYPlane, std::min(static_cast<size_t>(effYLinesize) * frameH, (size_t)4096)) ||
!isMemoryReadable(effUvPlane, std::min(static_cast<size_t>(effUvLinesize) * (frameH / 2), (size_t)4096))) {
return result;
}
uint8_t* dst = static_cast<uint8_t*>(m_pinnedBuf);
bool cpyOk = true;
if (effYLinesize == frameW) {
cpyOk = safeMemcpy(dst, effYPlane, ySize);
} else {
for (int row = 0; row < frameH && cpyOk; row++)
cpyOk = safeMemcpy(dst + row * frameW,
effYPlane + row * effYLinesize, frameW);
}
if (cpyOk) {
uint8_t* uvDst = dst + ySize;
if (effUvLinesize == frameW) {
cpyOk = safeMemcpy(uvDst, effUvPlane, uvSize);
} else {
for (int row = 0; row < frameH / 2 && cpyOk; row++)
cpyOk = safeMemcpy(uvDst + row * frameW,
effUvPlane + row * effUvLinesize, frameW);
}
}
if (!cpyOk) return result;
// (No registry lock to release — data kept alive by refcount)
cv::cuda::Stream cvStream = cv::cuda::StreamAccessor::wrapStream(stream);
cv::Mat pinnedY(frameH, frameW, CV_8UC1, m_pinnedBuf);
cv::Mat pinnedUV(frameH / 2, frameW, CV_8UC1,
static_cast<uint8_t*>(m_pinnedBuf) + ySize);
gpuY.upload(pinnedY, cvStream);
gpuUV.upload(pinnedUV, cvStream);
}
// Launch CHW kernel directly into TRT buffer
if (isFloat32) {
launchNV12ToRGBResizeCHW_float(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
static_cast<float*>(dstGpuBuffer),
inputW, inputH, frameW, frameH, stream);
} else {
launchNV12ToRGBResizeCHW_uint8(
gpuY.ptr<uint8_t>(), static_cast<int>(gpuY.step),
gpuUV.ptr<uint8_t>(), static_cast<int>(gpuUV.step),
static_cast<uint8_t*>(dstGpuBuffer),
inputW, inputH, frameW, frameH, stream);
}
cudaStreamSynchronize(stream);
// (No registry lock to release — data kept alive by refcount)
// Log activation once
if (!m_nv12ActiveLogged) {
m_nv12ActiveLogged = true;
const char* mode = useZeroCopy ? "CUDA zero-copy" : "CPU upload";
logger.LogInfo(std::string(tag) + "::NV12Helper",
std::string(mode) + " ACTIVE: " +
std::to_string(frameW) + "x" + std::to_string(frameH) +
" NV12 -> " + std::to_string(inputW) + "x" + std::to_string(inputH) +
" CHW " + (isFloat32 ? "float32" : "uint8"),
__FILE__, __LINE__);
}
result.succeeded = true;
result.metaWidth = static_cast<float>(inputImage.cols > 0 ? inputImage.cols : frameW);
result.metaHeight = static_cast<float>(inputImage.rows > 0 ? inputImage.rows : frameH);
return result;
}
} // namespace ANSCENTER
+226
View File
@@ -0,0 +1,226 @@
#ifndef NV12_PREPROCESS_HELPER_H
#define NV12_PREPROCESS_HELPER_H
#pragma once
// NV12PreprocessHelper — shared NV12/CUDA zero-copy preprocessing utility.
//
// Encapsulates all NV12 fast-path logic previously in ANSRTYOLO:
// - Warmup gating (skip NV12 for first N inferences)
// - Registry lookup + lock lifecycle
// - GPU matching (decode GPU vs inference GPU, cross-GPU BGR fallback)
// - Pinned buffer management (reusable cudaHostAlloc)
// - CUDA zero-copy path (wrap NVDEC device ptrs as GpuMat)
// - CPU NV12 upload path (SEH-protected memcpy + pinned upload)
// - CUDA health circuit breaker
// - BGR full-res path (pixFmt=1000)
// - One-shot diagnostic logging flags
//
// Composition, not inheritance — engines add it as a private member.
// CUDA coupling stays explicit and opt-in (ONNX/OpenVINO engines unaffected).
#include <functional>
#include <cuda_runtime.h>
#include <opencv2/core/cuda.hpp>
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h> // GetModuleHandleA, GetProcAddress for cross-DLL TLS
#endif
// ANSENGINE_API is defined by ANSEngineCommon.h (included before this header
// via ANSRTYOLO.h / ANSYOLOV12RTOD.h / ANSYOLOV10RTOD.h).
// Guard for standalone inclusion.
#ifndef ANSENGINE_API
#define ANSENGINE_API __declspec(dllexport)
#endif
// Forward declarations — avoid pulling in heavy headers
struct GpuFrameData;
// Thread-local: current GpuFrameData* for the active inference call.
// Set by RunInferenceComplete_LV (dllmain.cpp) before calling engine->RunInference().
// Read by tryNV12() and direct engine lookups instead of registry lookup by datastart.
//
// CROSS-DLL SHARING: The canonical thread_local lives in ANSODEngine.dll,
// exported via ANSODEngine_GetTlsGpuFrame(). All DLLs (ANSLPR, ANSFR,
// ANSOCR) resolve to that SAME slot via GetProcAddress at first call.
// This ensures that when ANSLPR sets the frame pointer, ANSODEngine's
// tryNV12() sees it — enabling NV12 zero-copy for ALPR, FR, and OCR.
//
// Previous bug: the old `inline` + `thread_local` created separate TLS
// instances per DLL on MSVC, silently disabling NV12 for all wrapper DLLs.
inline GpuFrameData*& tl_currentGpuFrame() {
// Resolve the canonical TLS slot exported by ANSODEngine.dll.
// GetProcAddress is called once (static lambda), cached forever.
using TlsFn = GpuFrameData** (*)();
static TlsFn s_fn = []() -> TlsFn {
#ifdef _WIN32
HMODULE h = GetModuleHandleA("ANSODEngine.dll");
if (h) {
auto f = reinterpret_cast<TlsFn>(
GetProcAddress(h, "ANSODEngine_GetTlsGpuFrame"));
if (f) return f;
}
#endif
return nullptr;
}();
if (s_fn) return *s_fn();
// Fallback: local TLS for unit tests that don't load ANSODEngine.dll.
thread_local GpuFrameData* local = nullptr;
return local;
}
namespace ANSCENTER {
// Forward declare SPDLogger (defined in ANSLicense.h, ANSCENTER namespace)
class SPDLogger;
// Callback type: engine-specific CUDA kernel launcher.
// Default YOLO launcher provided as static method.
using NV12KernelLauncher = std::function<void(
const cv::cuda::GpuMat& gpuY, const cv::cuda::GpuMat& gpuUV,
cv::cuda::GpuMat& gpuOut, int srcW, int srcH,
int inputW, int inputH, cudaStream_t stream)>;
// Result of tryNV12() — tells the caller what happened and what to do.
struct NV12Result {
bool succeeded = false; // true = gpuRGB is valid model input
cv::cuda::GpuMat gpuRGB; // model-input-sized, letterboxed RGB
float metaWidth = 0.f; // display-res width (for coord mapping)
float metaHeight = 0.f; // display-res height
float ratio = 1.f; // letterbox scale
bool useBgrFullRes = false; // true = pixFmt=1000 path
cv::Mat bgrFullResImg; // valid only when useBgrFullRes
float bgrFullResScaleX = 1.f; // display/fullRes X scale
float bgrFullResScaleY = 1.f; // display/fullRes Y scale
};
class ANSENGINE_API NV12PreprocessHelper {
public:
NV12PreprocessHelper() = default;
~NV12PreprocessHelper();
// Non-copyable (pinned buffer ownership)
NV12PreprocessHelper(const NV12PreprocessHelper&) = delete;
NV12PreprocessHelper& operator=(const NV12PreprocessHelper&) = delete;
// ── Main entry point ─────────────────────────────────────────────
NV12Result tryNV12(const cv::Mat& inputImage, int inferenceGpu,
int inputW, int inputH,
const NV12KernelLauncher& launcher,
SPDLogger& logger, const char* tag);
// Increment inference counter. Call after each inference (NV12 or BGR).
void tickInference() {
if (m_inferenceCount < NV12_WARMUP_THRESHOLD + 1)
++m_inferenceCount;
}
// ── CUDA context health ──────────────────────────────────────────
bool isCudaContextHealthy(SPDLogger& logger, const char* tag);
// ── Default YOLO kernel launcher ─────────────────────────────────
static NV12KernelLauncher defaultYOLOLauncher();
// ── Classification launcher (direct resize, no letterbox) ────────
static NV12KernelLauncher classificationLauncher();
// ── SCRFD center-padded letterbox launcher ───────────────────────
// SCRFD uses center-padding (dw, dh) unlike YOLO's right-bottom padding.
static NV12KernelLauncher scrfdCenterLetterboxLauncher(int padLeft, int padTop);
// ── NV12→BGR fused resize (for OCR detection) ────────────────────
// Single kernel: NV12 Y+UV → bilinear resize → BGR GpuMat output.
// No intermediate allocations. Output is CV_8UC3.
static void nv12ToBGRResize(
const uint8_t* devY, int yPitch,
const uint8_t* devUV, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH, int srcW, int srcH,
cudaStream_t stream = nullptr);
// ── NV12 affine warp for face alignment ─────────────────────────
// Reads full-res NV12, applies inverse affine transform, outputs
// aligned face in BGR (112x112). Used by SCRFD face pipeline.
struct NV12AffineResult {
bool succeeded = false;
cv::Mat alignedFaceBGR; // small face crop on CPU (e.g. 112x112 BGR)
cv::cuda::GpuMat gpuAlignedFace; // same face on GPU (avoids re-upload in recognizer)
};
NV12AffineResult tryNV12AffineWarp(
const cv::Mat& inputImage, int inferenceGpu,
const cv::Mat& affineMatrix, int outW, int outH,
float scaleX, float scaleY,
SPDLogger& logger, const char* tag);
// ── NV12 rectangular crop → BGR ──────────────────────────────────
// Crops a rectangular region from the full-res NV12 frame in GPU VRAM
// and returns a small BGR cv::Mat on CPU. Used by LPR pipeline to get
// high-res plate crops without converting the entire 4K frame.
// bbox is in display-res coords; scaleX/scaleY map to full-res NV12.
struct NV12CropResult {
bool succeeded = false;
cv::Mat bgrCrop; // cropped BGR on CPU (full-res quality)
};
NV12CropResult tryNV12CropToBGR(
const cv::Mat& inputImage, int inferenceGpu,
const cv::Rect& bbox, int padding,
float scaleX, float scaleY,
SPDLogger& logger, const char* tag);
// ── Direct-to-buffer CHW output (SAM3 image encoder) ─────────────
// Writes NV12→RGB + resize directly into a pre-allocated TRT GPU
// buffer in CHW planar format. No intermediate GpuMat allocation.
struct NV12DirectResult {
bool succeeded = false;
float metaWidth = 0.f;
float metaHeight = 0.f;
};
NV12DirectResult tryNV12DirectToBuffer(
const cv::Mat& inputImage, int inferenceGpu,
void* dstGpuBuffer, int inputW, int inputH,
bool isFloat32, // false=uint8 CHW, true=float32 CHW
cudaStream_t stream,
SPDLogger& logger, const char* tag);
// ── Cleanup ──────────────────────────────────────────────────────
void destroy();
private:
void ensurePinnedBuffer(size_t bytes, SPDLogger& logger, const char* tag);
// Warmup gating
int m_inferenceCount = 0;
static constexpr int NV12_WARMUP_THRESHOLD = 30;
// Pinned (page-locked) host buffer for fast H2D uploads
void* m_pinnedBuf = nullptr;
size_t m_pinnedBufSize = 0;
// CUDA context health circuit breaker
int m_cudaFailStreak = 0;
static constexpr int CUDA_FAIL_LIMIT = 10;
bool m_cudaContextDead = false;
// One-shot diagnostic logging flags
bool m_gpuMismatchLogged = false;
bool m_bgrFullResLogged = false;
bool m_nv12NullLogged = false;
bool m_nv12DimLogged = false;
bool m_nv12DeadLogged = false;
bool m_nv12PathLogged = false;
bool m_nv12PinnedLogged = false;
bool m_nv12ActiveLogged = false;
bool m_nv12MetaLogged = false;
bool m_nv12CropLogged = false;
};
} // namespace ANSCENTER
#endif
+279
View File
@@ -0,0 +1,279 @@
#include "RetinaFaceDetector.h"
#include "Utility.h"
namespace ANSCENTER {
bool ANSRETINAFD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
bool result = ANSFDBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
// We do not need to check for the license
_licenseValid = true;
if (!_licenseValid) return false;
try {
labelMap = "Face";
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::FACEDETECT;
_modelConfig.detectionType = DetectionType::FACEDETECTOR;
// We need to get the modelfolder from here
std::string onnxfile = CreateFilePath(_modelFolder, "retinaface.onnx");
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
this->_logger.LogDebug("ANSRETINAFD::Initialize. Loading retina weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSRETINAFD::Initialize. Model retinaface.onnx file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
std::string params_file;
std::string TrtCacheFile = CreateFilePath(_modelFolder, "retinaface.cache");;
auto option = fastdeploy::RuntimeOption();
// Check if the system has NVIDIA GPU
EngineType engineType = ANSLicenseHelper::CheckHardwareInformation();
if (engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
option.UseGpu();
option.UseTrtBackend();
option.trt_option.SetShape("images", { 1, 3, 640, 640 });
option.trt_option.enable_fp16 = false;
option.trt_option.serialize_file = TrtCacheFile;
}
else {
// Otherwise it use CPU
option.UseCpu();
option.UseOrtBackend();
}
auto format = fastdeploy::ModelFormat::ONNX;
this->_faceDectector = std::make_unique<fastdeploy::vision::facedet::RetinaFace>(onnxfile, params_file, option, format);
if (!_faceDectector->Initialized()) {
this->_logger.LogFatal("ANSRETINAFD::Initialize", "Failed to initialize face detector model", __FILE__, __LINE__);
return false;
}
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSRETINAFD::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSRETINAFD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
// We need to get the _modelFolder
bool result = ANSFDBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// We need to get the modelfolder from here
std::string onnxfile = CreateFilePath(_modelFolder, "retinaface.onnx");
if (std::filesystem::exists(onnxfile)) {
_modelFilePath = onnxfile;
this->_logger.LogDebug("ANSRETINAFD::LoadModel. Loading retinaface weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSRETINAFD::LoadModel. Model retinaface.onnx file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSRETINAFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSRETINAFD::OptimizeModel(bool fp16, std::string& optimizedModelFolder){
if (!FileExist(_modelFilePath)) {
optimizedModelFolder = "";
return false;
}
optimizedModelFolder = GetParentFolder(_modelFilePath);
std::string params_file;
std::string TrtCacheFile = CreateFilePath(_modelFolder, "retinaface.cache");;
auto option = fastdeploy::RuntimeOption();
EngineType engineType = ANSLicenseHelper::CheckHardwareInformation();
if (engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA
{
option.UseGpu();
option.UseTrtBackend();
option.trt_option.SetShape("images", { 1, 3, 640, 640 });
option.trt_option.enable_fp16 = fp16;
option.trt_option.serialize_file = TrtCacheFile;
}
else {
option.UseCpu();
option.UseOrtBackend();
}
auto format = fastdeploy::ModelFormat::ONNX;
auto model = fastdeploy::vision::facedet::RetinaFace(_modelFilePath, params_file, option, format);
if (!model.Initialized()) {
this->_logger.LogError("ANSRETINAFD::OptimizeModel.", "Failed to initialize", __FILE__, __LINE__);
return false;
}
return true;
}
std::vector<Object> ANSRETINAFD::RunInference(const cv::Mat& input) {
std::vector<Object> output;
output.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSRETINAFD::RunInference", "Invalid license", __FILE__, __LINE__);
return output;
}
if (!_isInitialized) {
this->_logger.LogError("ANSRETINAFD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return output;
}
try {
bool croppedFace = false; // Check if the image is cropped face image
cv::Mat im = input.clone();
// We know that the image sizes <=300 px, it is likely that image is cropped for face only
if ((input.size[0] <= 300) || (input.size[1] <= 300)) croppedFace = true;
if (croppedFace) cv::copyMakeBorder(input, im, 200, 200, 200, 200, cv::BORDER_REPLICATE);
fastdeploy::vision::FaceDetectionResult res;
if (!_faceDectector->Predict(&im, &res)) {
this->_logger.LogFatal("ANSRETINAFD::RunInference", "Failed to predict.", __FILE__, __LINE__);
return output;
}
if (res.boxes.size() > 0) {
// Peform face alignment
std::vector<cv::Mat> detectedFaces =
fastdeploy::vision::utils::AlignFaceWithFivePoints(im, res);
if (res.boxes.size() == detectedFaces.size()) {
for (int i = 0; i < res.boxes.size(); i++)
{
Object result;
float confidence = res.scores[i];
if (confidence >= _modelConfig.detectionScoreThreshold) {
int x_min = res.boxes[i][0];
int y_min = res.boxes[i][1];
int x_max = res.boxes[i][2];
int y_max = res.boxes[i][3];
result.classId = 0;
result.className = "Face";
result.confidence = confidence;
result.box.x = x_min;
result.box.y = y_min;
if (croppedFace) {
if (x_min <= 200) x_min = 200;
if (y_min <= 200) y_min = 200;
result.box.x = x_min - 200;
result.box.y = y_min - 200;
}
result.box.width = x_max - x_min;
result.box.height = y_max - y_min;
result.mask = detectedFaces.at(i).clone();
result.cameraId = "RETINAFACECAM";
output.push_back(result);
}
}
}
//detectedFaces.clear();
}
im.release();
res.Clear();
res.Free();
EnqueueDetection(output, "RETINAFACECAM");
return output;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSRETINAFD::RunInference", e.what(), __FILE__, __LINE__);
return output;
}
}
std::vector<Object> ANSRETINAFD::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::vector<Object> output;
output.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSRETINAFD::RunInference", "Invalid license", __FILE__, __LINE__);
return output;
}
if (!_isInitialized) {
this->_logger.LogError("ANSRETINAFD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return output;
}
try {
bool croppedFace = false; // Check if the image is cropped face image
cv::Mat im = input.clone();
// We know that the image sizes <=300 px, it is likely that image is cropped for face only
if ((input.size[0] <= 300) || (input.size[1] <= 300)) croppedFace = true;
if (croppedFace) cv::copyMakeBorder(input, im, 200, 200, 200, 200, cv::BORDER_REPLICATE);
fastdeploy::vision::FaceDetectionResult res;
if (!_faceDectector->Predict(&im, &res)) {
this->_logger.LogFatal("ANSRETINAFD::RunInference", "Failed to predict.", __FILE__, __LINE__);
return output;
}
if (res.boxes.size() > 0) {
// Peform face alignment
std::vector<cv::Mat> detectedFaces =
fastdeploy::vision::utils::AlignFaceWithFivePoints(im, res);
if (res.boxes.size() == detectedFaces.size()) {
for (int i = 0; i < res.boxes.size(); i++)
{
Object result;
float confidence = res.scores[i];
if (confidence >= _modelConfig.detectionScoreThreshold) {
int x_min = res.boxes[i][0];
int y_min = res.boxes[i][1];
int x_max = res.boxes[i][2];
int y_max = res.boxes[i][3];
result.classId = 0;
result.className = "Face";
result.confidence = confidence;
result.box.x = x_min;
result.box.y = y_min;
if (croppedFace) {
if (x_min <= 200) x_min = 200;
if (y_min <= 200) y_min = 200;
result.box.x = x_min - 200;
result.box.y = y_min - 200;
}
result.box.width = x_max - x_min;
result.box.height = y_max - y_min;
result.mask = detectedFaces.at(i).clone();
result.cameraId = camera_id;
output.push_back(result);
}
}
}
//detectedFaces.clear();
}
im.release();
res.Clear();
res.Free();
EnqueueDetection(output,camera_id);
return output;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSRETINAFD::RunInference", e.what(), __FILE__, __LINE__);
return output;
}
}
ANSRETINAFD::~ANSRETINAFD() {
try {
// if (_faceDectector) {
// delete _faceDectector;
//_faceDectector = nullptr;
// }
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSRETINAFD::Destroy", e.what(), __FILE__, __LINE__);
}
}
bool ANSRETINAFD::Destroy() {
try {
//if (_faceDectector) {
// delete _faceDectector;
// _faceDectector = nullptr;
//}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSRETINAFD::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
}
+30
View File
@@ -0,0 +1,30 @@
#ifndef RetinaFaceDetector_H
#define RetinaFaceDetector_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
#include "fastdeploy/vision.h"
namespace ANSCENTER {
class ANSENGINE_API ANSRETINAFD :public ANSFDBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id);
bool Destroy();
~ANSRETINAFD();
private:
ModelConfig _modelConfig;
std::string _modelFilePath;
std::unique_ptr<fastdeploy::vision::facedet::RetinaFace>_faceDectector = nullptr;
};
}
#endif
File diff suppressed because it is too large Load Diff
+112
View File
@@ -0,0 +1,112 @@
#ifndef SCRFDFaceDetector_H
#define SCRFDFaceDetector_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
#include "engine.h"
#include "engine/EnginePoolManager.h"
#include "NV12PreprocessHelper.h"
//#define FACEDEBUG
namespace ANSCENTER {
class ANSENGINE_API ANSSCRFDFD :public ANSFDBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
virtual bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap)override;
std::vector<Object> RunInference(const cv::Mat& input, bool useDynamicImage = true, bool validateFace=false, bool facelivenessCheck = true);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id, bool useDynamicImage = true, bool validateFace=false, bool facelivenessCheck = true);
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
bool Destroy();
~ANSSCRFDFD();
private:
std::string _modelFilePath;
std::recursive_mutex _mutex;
std::vector<Object> _movementObjects;
int _retainDetectedFaces{ 0 };
ANSCENTER::Options m_options;
NV12PreprocessHelper m_nv12Helper;
std::shared_ptr<Engine<float>> m_trtEngine = nullptr;
EnginePoolManager<float>::PoolKey m_poolKey;
bool m_usingSharedPool = false;
int m_maxSlotsPerGpu{ 1 }; // 1 = one slot per GPU, multi-GPU round-robin (no elastic)
void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; }
const std::array<float, 3> SUB_VALS{ 0.5f, 0.5f, 0.5f };
const std::array<float, 3> DIV_VALS{ 0.5f, 0.5f, 0.5f };
const bool NORMALIZE = true;
std::vector<Object> TensorRTInferene(const cv::Mat& input, const std::string& camera_id, bool useDynamicImage = false);
std::vector<Object> Detect(const cv::Mat& input);
std::vector<Object> Inference(const cv::Mat& input, const std::string& camera_id, bool DynamicImage = false, bool validateFace=false);
std::vector<Object> InferenceDynamic(const cv::Mat& input, const std::string& camera_id);
// Special variables for TensorRT
const int INPUT_H = 640;
const int INPUT_W = 640;
typedef struct
{
float cx;
float cy;
float stride;
} SCRFDPoint;
typedef struct
{
float ratio;
int dw;
int dh;
bool flag;
} SCRFDScaleParams;
unsigned int fmc = 3; // feature map count
bool use_kps = false;
unsigned int num_anchors = 2;
std::vector<int> feat_stride_fpn = { 8, 16, 32 }; // steps, may [8, 16, 32, 64, 128]
std::unordered_map<int, std::vector<SCRFDPoint>> center_points;
bool center_points_is_update = false;
static constexpr const unsigned int nms_pre = 1000;
static constexpr const unsigned int max_nms = 30000;
void resize_unscale(const cv::Mat& mat,
cv::Mat& mat_rs,
int target_height,
int target_width,
SCRFDScaleParams& scale_params);
// generate once.
void generate_points(const int target_height, const int target_width);
void generate_bboxes_single_stride(const SCRFDScaleParams& scale_params,
std::vector<float>& score_pred,
std::vector<float>& bbox_pred,
unsigned int stride,
float score_threshold,
float img_height,
float img_width,
std::vector<Object>& bbox_kps_collection);
void generate_bboxes_kps_single_stride(const SCRFDScaleParams& scale_params,
std::vector<float>& score_pred,
std::vector<float>& bbox_pred,
std::vector<float>& kps_pred,
unsigned int stride,
float score_threshold,
float img_height,
float img_width,
std::vector<Object>& bbox_kps_collection);
void generate_bboxes_kps(const SCRFDScaleParams& scale_params,
std::vector<Object>& bbox_kps_collection,
std::vector<std::vector<float>>& output_tensors,
float score_threshold, float img_height,
float img_width); // rescale & exclude
void nms_bboxes_kps(std::vector<Object>& input,
std::vector<Object>& output,
float iou_threshold, unsigned int topk);
float getIouOfObjects(const Object& a, const Object& b);
};
}
#endif
+345
View File
@@ -0,0 +1,345 @@
#include "SCRFDOVFaceDetector.h"
#include "Utility.h"
// Still not working
namespace ANSCENTER {
bool ANSOVSCRFDFD::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) {
bool result = ANSFDBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap);
labelMap = "Face";
// We do not need to check for the license
_licenseValid = true;
if (!_licenseValid) return false;
try {
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::FACEDETECT;
_modelConfig.detectionType = DetectionType::FACEDETECTOR;
// We need to get the modelfolder from here
std::string xmlfile = CreateFilePath(_modelFolder, "scrfdface.xml");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSOVSCRFDFD::Initialize. Loading SCRFD weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVSCRFDFD::Initialize. Model scrfdface.xml file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
ov::Core core;
std::shared_ptr<ov::Model> model = core.read_model(_modelFilePath);
//ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model);
const ov::Shape& inputShape = model->input().get_shape();
ov::Layout layout = ov::layout::get_layout(model->input());
if (layout.empty()) {
layout = getLayoutFromShape(model->input().get_partial_shape());
slog::warn << "Automatically detected layout '" << layout.to_string() << "' for input '"
<< model->input().get_any_name() << "' will be used." << slog::endl;
}
const ov::Layout& inputLayout = layout;
//ppp.input().tensor().set_element_type(ov::element::u8).set_layout("NHWC").set_color_format(ov::preprocess::ColorFormat::BGR);
//ppp.input().preprocess().convert_element_type(ov::element::f32).convert_color(ov::preprocess::ColorFormat::RGB).scale({ 255, 255, 255 });
//ppp.input().model().set_layout("NCHW");
////ppp.output().tensor().set_element_type(ov::element::f32);
ov::preprocess::PrePostProcessor ppp(model);
inputTransform.setPrecision(ppp, model->input().get_any_name());
ppp.input().tensor().set_layout({ "NHWC" });
ppp.input().model().set_layout(inputLayout);
// --------------------------- Reading image input parameters -------------------------------------------
inputsNames.push_back(model->input().get_any_name());
netInputWidth = inputShape[ov::layout::width_idx(inputLayout)];
netInputHeight = inputShape[ov::layout::height_idx(inputLayout)];
// --------------------------- Prepare output -----------------------------------------------------
int outputSize = model->outputs().size();
std::cout << "output Size:" << outputSize << std::endl;
const ov::Layout outputLayout{ "CHW" };
maxProposalsCount = model->outputs().front().get_shape()[ov::layout::height_idx(outputLayout)];
for (const auto& output : model->outputs()) {
const auto outTensorName = output.get_any_name();
outputsNames.push_back(outTensorName);
ppp.output(outTensorName).tensor().set_element_type(ov::element::f32).set_layout(outputLayout);
}
std::sort(outputsNames.begin(), outputsNames.end());
model = ppp.build();
compiled_model_ = core.compile_model(model, "GPU");
inference_request_ = compiled_model_.create_infer_request();
const std::vector<ov::Output<ov::Node>> inputs = model->inputs();
const ov::Shape input_shape = inputs[0].get_shape();
short height = input_shape[1];
short width = input_shape[2];
model_input_shape_ = cv::Size2f(width, height);
const std::vector<ov::Output<ov::Node>> outputs = model->outputs();
const ov::Shape output_shape = outputs[0].get_shape();
height = output_shape[1];
width = output_shape[2];
model_output_shape_ = cv::Size(width, height);
_isInitialized = true;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVSCRFDFD::Initialize", e.what(), __FILE__, __LINE__);
std::cout << "Error:" << e.what();
return false;
}
}
bool ANSOVSCRFDFD::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
// We need to get the _modelFolder
bool result = ANSFDBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
// We need to get the modelfolder from here
std::string xmlfile = CreateFilePath(_modelFolder, "scrfdface.xml");
if (std::filesystem::exists(xmlfile)) {
_modelFilePath = xmlfile;
this->_logger.LogDebug("ANSOVSCRFDFD::LoadModel. Loading scrfdface weight", _modelFilePath, __FILE__, __LINE__);
}
else {
this->_logger.LogError("ANSOVSCRFDFD::LoadModel. Model scrfdface.xml file is not exist", _modelFilePath, __FILE__, __LINE__);
return false;
}
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVSCRFDFD::LoadModel", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSOVSCRFDFD::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (!FileExist(_modelFilePath)) {
optimizedModelFolder = "";
return false;
}
return true;
}
cv::Rect ANSOVSCRFDFD::GetBoundingBox(const cv::Rect& src) {
cv::Rect box = src;
box.x = (box.x - 0.5 * box.width) * factor_.x;
box.y = (box.y - 0.5 * box.height) * factor_.y;
box.width *= factor_.x;
box.height *= factor_.y;
return box;
}
std::vector<Object>ANSOVSCRFDFD::PostProcessing() {
try {
std::vector<int> class_list;
std::vector<float> confidence_list;
std::vector<cv::Rect> box_list;
float* detections = inference_request_.get_output_tensor().data<float>();
const cv::Mat detection_outputs(model_output_shape_, CV_32F, (float*)detections);
for (int i = 0; i < detection_outputs.cols; ++i) {
const cv::Mat classes_scores = detection_outputs.col(i).rowRange(4, detection_outputs.rows);
cv::Point class_id;
double score;
cv::minMaxLoc(classes_scores, nullptr, &score, nullptr, &class_id);
if (score > 0.4) {
class_list.push_back(class_id.y);
confidence_list.push_back(score);
const float x = detection_outputs.at<float>(0, i);
const float y = detection_outputs.at<float>(1, i);
const float w = detection_outputs.at<float>(2, i);
const float h = detection_outputs.at<float>(3, i);
cv::Rect box;
box.x = static_cast<int>(x);
box.y = static_cast<int>(y);
box.width = static_cast<int>(w);
box.height = static_cast<int>(h);
box_list.push_back(box);
}
}
std::vector<int> NMS_result;
cv::dnn::NMSBoxes(box_list, confidence_list, 0.5, 0.5, NMS_result);
std::vector<Object> output;
for (int i = 0; i < NMS_result.size(); i++)
{
Object result;
int id = NMS_result[i];
result.classId = class_list[id];
result.confidence = confidence_list[id];
result.box = GetBoundingBox(box_list[id]);
output.push_back(result);
}
return output;
}
catch (const std::exception& e) {
std::vector<Object> result;
result.clear();
std::cout << "ANSOVDetector::PostprocessImage. " << e.what() << std::endl;
return result;
}
}
void ANSOVSCRFDFD::PreprocessImage(cv::Mat& frame) {
try {
cv::resize(frame, resized_frame_, model_input_shape_, 0, 0, cv::INTER_AREA);
factor_.x = static_cast<float>(frame.cols / model_input_shape_.width);
factor_.y = static_cast<float>(frame.rows / model_input_shape_.height);
float* input_data = (float*)resized_frame_.data;
input_tensor_ = ov::Tensor(compiled_model_.input().get_element_type(), compiled_model_.input().get_shape(), input_data);
inference_request_.set_input_tensor(input_tensor_);
}
catch (const std::exception& e) {
std::cout << "ANSOVSCRFDFD::PreprocessImage. " << e.what() << std::endl;
}
catch (...) {
std::cout << "ANSOVSCRFDFD::PreprocessImage. " << "unknown exception" << std::endl;
}
}
std::vector<Object> ANSOVSCRFDFD::RunInference(const cv::Mat& input) {
std::vector<Object> outputs;
outputs.clear();
if (!_licenseValid) {
this->_logger.LogError("ANSOVSCRFDFD::RunInference", "Invalid license", __FILE__, __LINE__);
return outputs;
}
if (!_isInitialized) {
this->_logger.LogError("ANSOVSCRFDFD::RunInference", "Model is not initialized", __FILE__, __LINE__);
return outputs;
}
try {
bool croppedFace = false; // Check if the image is cropped face image
cv::Mat im = input.clone();
//std::cout << "Width x Height=" << input.size[0] << "x" << input.size[1] << std::endl;
// We know that the image sizes <=300 px, it is likely that image is cropped for face only
if ((input.size[0] <= 300) || (input.size[1] <= 300)) croppedFace = true;
if (croppedFace) cv::copyMakeBorder(input, im, 200, 200, 200, 200, cv::BORDER_REPLICATE);
this->PreprocessImage(im);
// Perform inference
inference_request_.infer();
std::cout << "Model outputs:" << std::endl;
auto ovOutputs= compiled_model_.outputs();
for (int i = 0; i < ovOutputs.size();i++) {
auto output_tensor = inference_request_.get_output_tensor(i);
std::cout << "Output Tensor for " << ovOutputs[i].get_any_name() << " obtained." << std::endl;
}
auto scores_tensor = inference_request_.get_output_tensor(1);
auto scores_shape = scores_tensor.get_shape();
// Access the data from the tensor
float* scores_data = scores_tensor.data<float>();
int num_detections = scores_shape[2]; // Should be 800
// Define a confidence threshold
float confidence_threshold = 0.1;
// Collecting indices of valid detections
std::vector<int> valid_detections;
for (int i = 0; i < num_detections; i++) {
float score = scores_data[i];
/* std::cout << "score [" << i << "]=" << score << std:: endl;*/
if (score > confidence_threshold) {
valid_detections.push_back(i);
//std::cout << "Detection " << i << " is valid with a score of " << score << std::endl;
}
}
std::cout << "Total valid detections: " << valid_detections.size() << std::endl;
/* float* score = inference_request_.get_output_tensor(2).data<float>();
//if (score != nullptr) {
// std::cout << "Score:" << score[0]<<std::endl;
//}
float* bbox = inference_request_.get_output_tensor(5).data<float>();*/
// Get the output tensor
//// Step 0: Prepare input
//this->PreprocessImage(im);
////Synchronous mode
//inference_request_.infer();
//std::vector<Object> output = PostProcessing();
//if (res.size() > 0)
//{
// // Peform face alignment
// //std::vector<cv::Mat> detectedFaces =fastdeploy::vision::utils::AlignFaceWithFivePoints(im, res);
// std::vector<cv::Mat> detectedFaces = ANSUtilityHelper::AlignFaceWithFivePoints(im, res.boxes, res.landmarks);
// if (res.size() == detectedFaces.size())
// {
// for (int i = 0; i < res.boxes.size(); i++)
// {
// Object result;
// float confidence = res.scores[i];
// if (confidence > _modelConfig.detectionScoreThreshold)
// {
// int x_min = res.boxes[i][0];
// int y_min = res.boxes[i][1];
// int x_max = res.boxes[i][2];
// int y_max = res.boxes[i][3];
// result.classId = 0;
// result.className = "Face";
// result.confidence = confidence;
// result.box.x = x_min;
// result.box.y = y_min;
// if (croppedFace) {
// if (x_min <= 200) x_min = 200;
// if (y_min <= 200) y_min = 200;
// result.box.x = x_min - 200;
// result.box.y = y_min - 200;
// }
// result.box.width = x_max - x_min;
// result.box.height = y_max - y_min;
// result.mask = detectedFaces.at(i).clone();
// output.push_back(result);
// }
// }
// }
// //detectedFaces.clear();
//}
im.release();
return outputs;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVSCRFDFD::RunInference", e.what(), __FILE__, __LINE__);
return outputs;
}
}
ANSOVSCRFDFD::~ANSOVSCRFDFD() {
try {
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSOVSCRFDFD::Destroy", e.what(), __FILE__, __LINE__);
}
}
bool ANSOVSCRFDFD::Destroy() {
return true;
}
}
+47
View File
@@ -0,0 +1,47 @@
#ifndef SCRFDOVFaceDetector_H
#define SCRFDOVFaceDetector_H
#pragma once
#include <string.h>
#include <vector>
#include <iostream>
#include <typeinfo>
#include "ANSEngineCommon.h"
namespace ANSCENTER {
class ANSENGINE_API ANSOVSCRFDFD :public ANSFDBase {
public:
virtual bool Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) override;
virtual bool LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword)override;
bool OptimizeModel(bool fp16, std::string& optimizedModelFolder);
std::vector<Object> RunInference(const cv::Mat& input);
std::vector<Object> RunInference(const cv::Mat& input, const std::string& camera_id) {
// Not implemented for future use
return RunInference(input);
}
bool Destroy();
~ANSOVSCRFDFD();
private:
ModelConfig _modelConfig;
std::string _modelFilePath;
cv::Mat resized_frame_;
cv::Point2f factor_;
cv::Size2f model_input_shape_;
cv::Size model_output_shape_;
ov::Tensor input_tensor_;
ov::InferRequest inference_request_;
ov::CompiledModel compiled_model_;
cv::Rect GetBoundingBox(const cv::Rect& src);
void PreprocessImage(cv::Mat& frame);
std::vector<Object> PostProcessing();
InputTransform inputTransform = InputTransform();
std::vector<std::string> inputsNames;
std::vector<std::string> outputsNames;
size_t netInputHeight = 0;
size_t netInputWidth = 0;
size_t maxProposalsCount;
};
}
#endif
File diff suppressed because it is too large Load Diff
+17
View File
@@ -0,0 +1,17 @@
"""Download CLIP BPE merges.txt for ANSSAM3 tokenizer."""
import urllib.request
import sys
import os
URL = "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt"
def main():
out_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.abspath(__file__))
out_path = os.path.join(out_dir, "merges.txt")
print(f"Downloading merges.txt to {out_path} ...")
urllib.request.urlretrieve(URL, out_path)
print(f"Done ({os.path.getsize(out_path)} bytes)")
if __name__ == "__main__":
main()
+161
View File
@@ -0,0 +1,161 @@
#include "engine.h"
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <random>
#include "macros.h"
#include "logger.h"
#include "util/Stopwatch.h"
#include "Utility.h"
using namespace nvinfer1;
using namespace Util;
void Logger::log(Severity severity, const char *msg) noexcept {
switch (severity) {
case Severity::kVERBOSE:
spdlog::debug(msg);
break;
case Severity::kINFO:
spdlog::info(msg);
break;
case Severity::kWARNING:
spdlog::warn(msg);
break;
case Severity::kERROR:
spdlog::error(msg);
break;
case Severity::kINTERNAL_ERROR:
spdlog::critical(msg);
break;
default:
spdlog::info("Unexpected severity level");
}
}
Int8EntropyCalibrator2::Int8EntropyCalibrator2(int32_t batchSize, int32_t inputW, int32_t inputH, const std::string &calibDataDirPath,
const std::string &calibTableName, const std::string &inputBlobName,
const std::array<float, 3> &subVals, const std::array<float, 3> &divVals, bool normalize,
bool readCache)
: m_batchSize(batchSize), m_inputW(inputW), m_inputH(inputH), m_imgIdx(0), m_calibTableName(calibTableName),
m_inputBlobName(inputBlobName), m_subVals(subVals), m_divVals(divVals), m_normalize(normalize), m_readCache(readCache) {
// Allocate GPU memory to hold the entire batch
m_inputCount = 3 * inputW * inputH * batchSize;
checkCudaErrorCode(cudaMalloc(&m_deviceInput, m_inputCount * sizeof(float)));
// Read the name of all the files in the specified directory.
if (!doesFileExist(calibDataDirPath)) {
auto msg = "Error, directory at provided path does not exist: " + calibDataDirPath;
spdlog::error(msg);
//throw std::runtime_error(msg);
}
m_imgPaths = getFilesInDirectory(calibDataDirPath);
if (m_imgPaths.size() < static_cast<size_t>(batchSize)) {
auto msg = "Error, there are fewer calibration images than the specified batch size!";
spdlog::error(msg);
//throw std::runtime_error(msg);
}
// Randomize the calibration data
auto rd = std::random_device{};
auto rng = std::default_random_engine{rd()};
std::shuffle(std::begin(m_imgPaths), std::end(m_imgPaths), rng);
}
int32_t Int8EntropyCalibrator2::getBatchSize() const noexcept {
// Return the batch size
return m_batchSize;
}
bool Int8EntropyCalibrator2::getBatch(void **bindings, const char **names, int32_t nbBindings) noexcept {
// This method will read a batch of images into GPU memory, and place the
// pointer to the GPU memory in the bindings variable.
if (m_imgIdx + m_batchSize > static_cast<int>(m_imgPaths.size())) {
// There are not enough images left to satisfy an entire batch
return false;
}
// Read the calibration images into memory for the current batch
std::vector<cv::cuda::GpuMat> inputImgs;
for (int i = m_imgIdx; i < m_imgIdx + m_batchSize; i++) {
spdlog::info("Reading image {}: {}", i, m_imgPaths[i]);
auto cpuImg = cv::imread(m_imgPaths[i]);
if (cpuImg.empty()) {
spdlog::error("Fatal error: Unable to read image at path: " + m_imgPaths[i]);
return false;
}
cv::cuda::Stream stream; // Create a custom stream
cv::cuda::GpuMat gpuImg;
gpuImg.upload(cpuImg, stream);
stream.waitForCompletion();
//cv::cuda::cvtColor(gpuImg, gpuImg, cv::COLOR_BGR2RGB);
// TODO: Define any preprocessing code here, such as resizing
auto resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(gpuImg, m_inputH, m_inputW);
inputImgs.emplace_back(std::move(resized));
}
// Convert the batch from NHWC to NCHW
// ALso apply normalization, scaling, and mean subtraction
auto mfloat = Engine<float>::blobFromGpuMats(inputImgs, m_subVals, m_divVals, m_normalize, true);
auto *dataPointer = mfloat.ptr<void>();
// Copy the GPU buffer to member variable so that it persists
checkCudaErrorCode(cudaMemcpyAsync(m_deviceInput, dataPointer, m_inputCount * sizeof(float), cudaMemcpyDeviceToDevice));
m_imgIdx += m_batchSize;
if (std::string(names[0]) != m_inputBlobName) {
spdlog::error("Error: Incorrect input name provided!");
return false;
}
bindings[0] = m_deviceInput;
return true;
}
void const *Int8EntropyCalibrator2::readCalibrationCache(size_t &length) noexcept {
spdlog::info("Searching for calibration cache: {}", m_calibTableName);
m_calibCache.clear();
std::ifstream input(m_calibTableName, std::ios::binary);
input >> std::noskipws;
if (m_readCache && input.good()) {
spdlog::info("Reading calibration cache: {}", m_calibTableName);
std::copy(std::istream_iterator<char>(input), std::istream_iterator<char>(), std::back_inserter(m_calibCache));
}
length = m_calibCache.size();
return length ? m_calibCache.data() : nullptr;
}
void Int8EntropyCalibrator2::writeCalibrationCache(const void *ptr, std::size_t length) noexcept {
spdlog::info("Writing calibration cache: {}", m_calibTableName);
spdlog::info("Calibration cache size: {} bytes", length);
std::ofstream output(m_calibTableName, std::ios::binary);
output.write(reinterpret_cast<const char *>(ptr), length);
}
Int8EntropyCalibrator2::~Int8EntropyCalibrator2() { checkCudaErrorCode(cudaFree(m_deviceInput)); };
// ============================================================================
// Explicit template instantiation definitions — DLL export
//
// Each line here:
// 1. Forces the compiler to generate every method of Engine<T> in THIS
// translation unit (engine.obj).
// 2. Marks all those generated symbols __declspec(dllexport) so the linker
// includes them in the DLL's export table.
//
// Consuming projects include engine.h, which contains the matching
// extern template class ENGINE_API Engine<float>;
// declaration. That declaration suppresses re-instantiation in the consumer's
// TUs and directs the linker to resolve Engine<float> from this DLL.
//
// To export additional specialisations (e.g. Engine<uint8_t> for INT8 output),
// add a line here and the corresponding extern declaration in engine.h.
// ============================================================================
template class __declspec(dllexport) Engine<float>;
+810
View File
@@ -0,0 +1,810 @@
#pragma once
#include "engine_api.h" // ENGINE_API (__declspec dllexport / dllimport)
// -- TRT / ONNX-parser API decoration override -----------------------------
// Must appear BEFORE the first inclusion of NvInfer.h / NvOnnxParser.h.
//
// By default these macros expand to __declspec(dllimport) for consumers of
// the TRT DLL, which causes the compiler to emit indirect import-table calls.
// Overriding them to empty makes all calls direct, so the linker resolves
// them against the extern "C" stub definitions in NvDynLoader.cpp instead of
// against nvinfer_XX.lib / nvonnxparser_XX.lib.
//
// Remove nvinfer_XX.lib and nvonnxparser_XX.lib from Linker -> Input.
// Link ANSLibsLoader.lib instead. See NvDynLoader.h for full details.
#ifndef TENSORRTAPI
# define TENSORRTAPI
#endif
#ifndef NVONNXPARSER_API
# define NVONNXPARSER_API
#endif
#include "NvOnnxParser.h"
#include <atomic>
#include <chrono>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <deque>
#include <filesystem>
#include <fstream>
#include <unordered_map>
#include <memory>
#include <condition_variable>
#include <mutex>
#include <string>
#include <thread>
#include <opencv2/core/cuda.hpp>
#include <opencv2/core/cuda_stream_accessor.hpp>
#include <opencv2/cudaarithm.hpp>
#include "engine/TRTEngineCache.h"
#include <opencv2/cudaimgproc.hpp>
#include <opencv2/cudawarping.hpp>
#include <opencv2/opencv.hpp>
#include "interfaces\IEngine.h"
#include "ANSLicense.h"
#include "Int8Calibrator.h"
#include "NvDynLoader.h" // from ANSLibsLoader/include/
#ifdef _WIN32
#include <windows.h>
#endif
// Class to extend TensorRT logger
class ENGINE_API Logger : public nvinfer1::ILogger {
void log(Severity severity, const char *msg) noexcept override;
};
// ============================================================================
// Diagnostic logging to Windows Event Viewer (ANSLogger source).
// Allows internal engine failure points to be traced even when running
// under LabView which swallows stdout/stderr.
// ============================================================================
inline void logEngineEvent(const std::string& msg, bool isError = false) {
#ifdef _WIN32
// Cache the event source handle to avoid RegisterEventSourceA/
// DeregisterEventSource churn. Under heavy load (10+ AI tasks),
// the per-message register/deregister cycle overwhelms the Windows
// Event Log service, causing ERROR_UNRECOGNIZED_VOLUME (1005) floods
// that freeze the system.
static HANDLE hLog = RegisterEventSourceA(nullptr, "ANSLogger");
if (hLog) {
const char* msgs[1] = { msg.c_str() };
ReportEventA(hLog,
isError ? EVENTLOG_ERROR_TYPE : EVENTLOG_INFORMATION_TYPE,
0, isError ? 1004 : 1000, nullptr,
1, 0, msgs, nullptr);
}
#endif
if (isError)
std::cerr << msg << std::endl;
else
std::cout << msg << std::endl;
}
// ============================================================================
// GpuDeviceInfo -- snapshot of one CUDA device captured at pool-init time.
// Returned by Engine<T>::enumerateDevices() and stored inside the pool.
// ============================================================================
struct ENGINE_API GpuDeviceInfo {
int index = 0;
std::string name;
size_t totalMemoryBytes = 0; ///< Physical VRAM
size_t freeMemoryAtInitBytes = 0; ///< Free VRAM when pool started
int computeMajor = 0;
int computeMinor = 0;
int slotsAllocated = 0; ///< Engine<T> instances on this GPU
size_t memoryPerSlotBytes = 0; ///< VRAM bytes each slot occupies
};
template <typename T>
class Engine : public IEngine<T> {
public:
Engine();
Engine(const ANSCENTER::Options &options);
~Engine();
// Build the onnx model into a TensorRT engine file, cache the model to disk
// (to avoid rebuilding in future), and then load the model into memory The
// default implementation will normalize values between [0.f, 1.f] Setting the
// normalize flag to false will leave values between [0.f, 255.f] (some
// converted models may require this). If the model requires values to be
// normalized between [-1.f, 1.f], use the following params:
// subVals = {0.5f, 0.5f, 0.5f};
// divVals = {0.5f, 0.5f, 0.5f};
// normalize = true;
bool buildLoadNetwork(std::string onnxModelPath, const std::array<float, 3> &subVals = {0.f, 0.f, 0.f},
const std::array<float, 3> &divVals = {1.f, 1.f, 1.f}, bool normalize = true) override;
// Load a TensorRT engine file from disk into memory
// The default implementation will normalize values between [0.f, 1.f]
// Setting the normalize flag to false will leave values between [0.f, 255.f]
// (some converted models may require this). If the model requires values to
// be normalized between [-1.f, 1.f], use the following params:
// subVals = {0.5f, 0.5f, 0.5f};
// divVals = {0.5f, 0.5f, 0.5f};
// normalize = true;
bool loadNetwork(std::string trtModelPath, const std::array<float, 3> &subVals = {0.f, 0.f, 0.f},
const std::array<float, 3> &divVals = {1.f, 1.f, 1.f}, bool normalize = true) override;
// -- Multi-GPU pool overloads (non-virtual; call through Engine<T>* or
// unique_ptr<Engine<T>>, not through IEngine<T>*).
//
// maxSlotsPerGpu == 0 -> BYPASS: identical to the 4-param single-GPU path
// above. Used by OptimizeModelStr for temporary engines.
// maxSlotsPerGpu == 1 -> ROUND-ROBIN (default): 1 slot per GPU, multi-GPU
// round-robin dispatch. Tasks queue when all slots
// busy. Best balance of VRAM usage and multi-GPU
// utilisation.
// maxSlotsPerGpu == -1 -> ELASTIC pool: 1 probe slot at init, additional
// slots created on-demand when concurrent requests
// arrive, released when idle via releaseIdleSlots().
// Higher throughput but higher VRAM usage — only for
// large GPUs (≥ 8 GB).
// maxSlotsPerGpu > 1 -> PRE-ALLOCATED pool: cap at that many slots/GPU,
// all created at init time.
//
// Usage -- opt into pool mode by adding ONE extra argument:
// m_trtEngine->buildLoadNetwork(path, sub, div, norm); // single-GPU (unchanged)
// m_trtEngine->buildLoadNetwork(path, sub, div, norm, 1); // round-robin (default)
// m_trtEngine->buildLoadNetwork(path, sub, div, norm, -1); // elastic pool
// m_trtEngine->buildLoadNetwork(path, sub, div, norm, 4); // pre-allocated, 4 slots/GPU
bool buildLoadNetwork(std::string onnxModelPath,
const std::array<float, 3> &subVals,
const std::array<float, 3> &divVals,
bool normalize,
int maxSlotsPerGpu,
double memSafetyFactor = 0.80);
bool loadNetwork(std::string trtModelPath,
const std::array<float, 3> &subVals,
const std::array<float, 3> &divVals,
bool normalize,
int maxSlotsPerGpu,
double memSafetyFactor = 0.80);
// Run inference.
// Input format [input][batch][cv::cuda::GpuMat]
// Output format [batch][output][feature_vector]
bool runInference(const std::vector<std::vector<cv::cuda::GpuMat>> &inputs, std::vector<std::vector<std::vector<T>>> &featureVectors) override;
// Utility method for resizing an image while maintaining the aspect ratio by
// adding padding to smaller dimension after scaling While letterbox padding
// normally adds padding to top & bottom, or left & right sides, this
// implementation only adds padding to the right or bottom side This is done
// so that it's easier to convert detected coordinates (ex. YOLO model) back
// to the original reference frame.
static cv::cuda::GpuMat resizeKeepAspectRatioPadRightBottom(const cv::cuda::GpuMat &input, size_t height, size_t width,
const cv::Scalar &bgcolor = cv::Scalar(0, 0, 0));
[[nodiscard]] const std::vector<nvinfer1::Dims3> &getInputDims() const override { return m_inputDims; };
[[nodiscard]] const std::vector<nvinfer1::Dims> &getOutputDims() const override { return m_outputDims; };
[[nodiscard]] const ANSCENTER::Options& getOptions() const { return m_options; }
/// Return the CUDA device index that preprocessing should target.
/// In pool mode this returns the device of the first alive slot;
/// in single-engine mode it returns m_options.deviceIndex.
/// Call cudaSetDevice(engine->getPreferredDeviceIndex()) before
/// any cv::cuda::GpuMat operations to ensure multi-GPU correctness.
[[nodiscard]] int getPreferredDeviceIndex() const {
// Pool mode: use the first alive slot's device
if (!m_slots.empty()) {
for (const auto& s : m_slots) {
if (s.engine) return s.deviceIndex;
}
}
return m_options.deviceIndex;
}
/// Convenience: set the calling thread's CUDA device to match this engine.
/// Skips the call if already on the correct device (WDDM overhead avoidance).
void setDeviceContext() const {
const int target = getPreferredDeviceIndex();
int current = -1;
cudaGetDevice(&current);
if (current != target) {
cudaSetDevice(target);
}
}
// Utility method for transforming triple nested output array into 2D array
// Should be used when the output batch size is 1, but there are multiple
// output feature vectors
static void transformOutput(std::vector<std::vector<std::vector<T>>> &input, std::vector<std::vector<T>> &output);
// Utility method for transforming triple nested output array into single
// array Should be used when the output batch size is 1, and there is only a
// single output feature vector
static void transformOutput(std::vector<std::vector<std::vector<T>>> &input, std::vector<T> &output);
// Convert NHWC to NCHW and apply scaling and mean subtraction.
// Pass a cv::cuda::Stream to run all GPU kernels on that stream instead of
// the CUDA null (default) stream. This eliminates global sync barriers and
// enables CUDA-graph-safe preprocessing on OpenCV 4.13+.
static cv::cuda::GpuMat blobFromGpuMats(const std::vector<cv::cuda::GpuMat> &batchInput, const std::array<float, 3> &subVals,
const std::array<float, 3> &divVals, bool normalize, bool swapRB = false,
cv::cuda::Stream &stream = cv::cuda::Stream::Null());
// Converts the engine options into a string
std::string serializeEngineOptions(const ANSCENTER::Options& options, const std::string& onnxModelPath);
// Build the network (single attempt)
bool build(std::string onnxModelPath, const std::array<float, 3>& subVals, const std::array<float, 3>& divVals, bool normalize);
// SEH-safe wrapper around build(). Catches access violations that corrupt
// the CUDA context (e.g. opset 19+ ONNX models crash TRT's parser).
// Returns false and sets *outSehCode on crash.
bool buildSafe(std::string onnxModelPath, const std::array<float, 3>& subVals, const std::array<float, 3>& divVals, bool normalize, unsigned long* outSehCode);
// Build with auto-retry for dynamic spatial dimension models.
// Pre-analyzes the ONNX model; if H/W are dynamic and the max profile
// is too large for GPU VRAM, automatically reduces max spatial dims and
// retries until build succeeds or all candidates are exhausted.
// Fixed-spatial models fall through to a single build() call.
bool buildWithRetry(std::string onnxModelPath, const std::array<float, 3>& subVals, const std::array<float, 3>& divVals, bool normalize);
void getDeviceNames(std::vector<std::string>& deviceNames);
void clearGpuBuffers();
int getBindingIndexByName(const std::string& name);
void UpdateOptions(const ANSCENTER::Options& options) { m_options = options; }
void warmUp(int iterations = 5);
void setVerbose(bool v) { m_verbose = v; }
void setDisableGraphs(bool v) { m_disableGraphs = v; }
// -- Multi-GPU pool management (replaces MultiGpuEngineManager<T>) ---------
//
// Call ONE of the two initialize methods below instead of
// buildLoadNetwork() / loadNetwork(), then use runInference() exactly as
// you would for single-GPU. runInference() dispatches automatically to
// the first idle slot across all GPUs.
//
// Single-GPU code that calls buildLoadNetwork() + runInference() directly
// is completely unaffected -- it follows the existing code path unchanged.
/**
* Build from an ONNX file and create an Engine pool across all GPUs.
*
* @param baseOptions Configuration template; deviceIndex is ignored
* and overridden per slot.
* @param onnxModelPath Path to the .onnx model.
* @param subVals Per-channel subtraction for normalisation.
* @param divVals Per-channel divisor for normalisation.
* @param normalize Scale pixel values to [0, 1] before inference.
* @param maxSlotsPerGpu Cap slots per GPU (-1 = memory-limited only).
* @param memSafetyFactor Fraction of free VRAM to consume (default 0.80).
*/
bool initializePool(const ANSCENTER::Options& baseOptions,
const std::string& onnxModelPath,
const std::array<float, 3>& subVals = {0.f, 0.f, 0.f},
const std::array<float, 3>& divVals = {1.f, 1.f, 1.f},
bool normalize = true,
int maxSlotsPerGpu = -1,
double memSafetyFactor = 0.80);
/**
* Load a pre-built TRT engine and create a pool -- no ONNX build step.
*/
bool initializePoolFromEngine(const ANSCENTER::Options& baseOptions,
const std::string& trtEnginePath,
const std::array<float, 3>& subVals = {0.f, 0.f, 0.f},
const std::array<float, 3>& divVals = {1.f, 1.f, 1.f},
bool normalize = true,
int maxSlotsPerGpu = -1,
double memSafetyFactor = 0.80);
/** Enumerate all CUDA-capable GPUs without loading any model. */
static std::vector<GpuDeviceInfo> enumerateDevices();
/** Total Engine<T> slots across all GPUs (valid after initializePool). */
int getTotalCapacity() const { return m_totalCapacity; }
/** Slots currently executing inference (approximate, lock-free). */
int getActiveInferences() const { return m_activeCount.load(); }
/** Idle slots not currently running inference. */
int getAvailableSlots() const { return m_totalCapacity - m_activeCount.load(); }
/** True when every alive slot is busy.
* In elastic mode, new work will trigger on-demand slot creation
* rather than being rejected (unless all GPUs are physically full). */
bool isAtCapacity() const { return getAvailableSlots() <= 0; }
/** Device snapshots captured at initializePool() time. */
const std::vector<GpuDeviceInfo>& getDeviceInfo() const { return m_deviceInfos; }
/** Print a human-readable pool capacity and device report. */
void printCapacityReport() const;
/**
* Release idle slots whose last inference was more than @p idleSeconds
* ago. The probe slot (first slot) is never released so the model
* remains instantly usable. Returns the number of slots released.
*
* Typical usage -- call periodically from a background timer:
* @code
* int freed = engine->releaseIdleSlots(30.0);
* @endcode
*/
int releaseIdleSlots(double idleSeconds = 30.0);
/** Actual max spatial dims from the loaded engine's optimization profile.
* Returns -1 if the engine is not loaded yet or dim is fixed. */
int getProfileMaxHeight() const { return m_profileMaxHeight; }
int getProfileMaxWidth() const { return m_profileMaxWidth; }
/// Grow the pool by up to @p count slots (demand-driven, bypasses grace period).
/// Returns the number of slots actually created.
int growPool(int count);
private:
// Serialises concurrent runInference() calls on the SAME Engine instance.
// Without this, two threads with different batch sizes corrupt m_buffers,
// m_lastBatchSize, and the TRT execution context — causing fatal "illegal
// memory access" errors that permanently destroy the CUDA context.
// Pool-mode slots have their own busy-flag dispatch, so this only matters
// for the single-engine path (m_slots.empty()).
mutable std::mutex m_inferenceMutex;
int32_t m_lastBatchSize = -1;
// BATCH STATE MANAGEMENT (THREAD-SAFE)
//mutable std::mutex m_batchStateMutex; // Protects all batch-related state
//size_t m_targetStableBatchSize = 4; // Default optimal batch size
//int m_batchSizeChangeCounter = 0;
//const int BATCH_STABILITY_THRESHOLD = 15; // Frames before allowing downsize
//std::deque<size_t> m_recentBatchSizes;
//const size_t BATCH_HISTORY_SIZE = 30;
//std::chrono::steady_clock::time_point m_lastBatchSizeChange;
//// Context reshape mutex (separate from batch state)
//mutable std::mutex m_contextReshapeMutex;
//// Helper methods
//size_t DetermineOptimalBatchSize(size_t requestedSize);
//std::vector<std::vector<cv::cuda::GpuMat>> PadBatchToSize(const std::vector<std::vector<cv::cuda::GpuMat>>& inputs,size_t targetSize);
// Normalization, scaling, and mean subtraction of inputs
std::array<float, 3> m_subVals{};
std::array<float, 3> m_divVals{};
bool m_normalize;
// Holds pointers to the input and output GPU buffers
std::vector<void *> m_buffers;
std::vector<uint32_t> m_outputLengths{};
std::vector<nvinfer1::Dims3> m_inputDims;
std::vector<nvinfer1::Dims> m_outputDims;
std::vector<std::string> m_IOTensorNames;
int32_t m_inputBatchSize = 0;
bool m_firstInferenceCall = true; // Per-instance first-call diagnostics flag
bool m_batchShapeChangeLogged = false; // Suppress per-tensor shape dump after first batch-size change
bool m_hasDynamicSpatialDims = false; // True if any input has dynamic H or W (-1)
std::vector<nvinfer1::Dims> m_lastSetInputDims; // Cache of last setInputShape() dims per tensor — skip if unchanged
int m_profileMaxHeight = -1; // Max H from optimization profile (after load)
int m_profileMaxWidth = -1; // Max W from optimization profile (after load)
// Must keep IRuntime around for inference, see:
// https://forums.developer.nvidia.com/t/is-it-safe-to-deallocate-nvinfer1-iruntime-after-creating-an-nvinfer1-icudaengine-but-before-running-inference-with-said-icudaengine/255381/2?u=cyruspk4w6
//
// m_runtime and m_engine are shared_ptr to support TRTEngineCache:
// multiple Engine<T> instances loading the same model share one ICudaEngine
// in VRAM, each with its own IExecutionContext.
std::shared_ptr<nvinfer1::IRuntime> m_runtime = nullptr;
std::unique_ptr<Int8EntropyCalibrator2> m_calibrator = nullptr;
std::shared_ptr<nvinfer1::ICudaEngine> m_engine = nullptr;
std::unique_ptr<nvinfer1::IExecutionContext> m_context = nullptr;
// TRTEngineCache tracking — used by destructor to release shared engine
std::string m_cachedEnginePath;
int m_cachedGpuIndex = -1;
bool m_usingCachedEngine = false;
bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr)
bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth)
bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth)
bool m_lastLoadFailedVRAM = false; // set by loadNetwork when failure is due to insufficient VRAM
bool m_forceNoPool = false; // true = force single-GPU path, skip multi-GPU pool
ANSCENTER::Options m_options;
cudaStream_t m_cudaStream = nullptr;
cudaStream_t m_inferenceStream;
bool m_streamInitialized;
//int32_t m_lastBatchSize;
cudaStream_t m_memoryStream; // ADD THIS - separate stream for memory operations
std::vector<cv::cuda::GpuMat> m_preprocessedInputs; // Keep inputs alive
// Thermal management (ADD THESE)
//int m_consecutiveInferences;
//int m_thermalCooldownThreshold;
//int m_thermalCooldownDelayMs;
bool m_profileInitialized;
// -- GPU clock lock (NVML) -------------------------------------------------
// Dynamically loaded NVML to lock GPU clocks and prevent power throttling.
// No build-time dependency on nvml.lib — loads nvml.dll at runtime.
bool m_clocksLocked = false;
unsigned int m_nvmlDeviceIdx = 0;
void* m_nvmlLib = nullptr; // LibHandle (HMODULE on Windows)
void lockGpuClocks(int deviceIndex, int requestedMHz);
void unlockGpuClocks();
// -- CUDA Graph acceleration ---------------------------------------------
// Pinned (page-locked) host buffers -- stable addresses required for graph
// D2H copies. One pointer per output tensor, sized for maxBatchSize elements.
std::vector<T*> m_pinnedOutputBuffers;
std::vector<size_t> m_pinnedOutputBufElems; // elements per buffer
// Cached executable graphs, keyed by batch size. A new graph is captured
// the first time each batch size is seen; subsequent calls reuse it.
std::unordered_map<int, cudaGraphExec_t> m_graphExecs;
Logger m_logger;
bool m_verbose{ true }; // false for non-probe pool slots
bool m_disableGraphs{ false }; // true for pool slots — concurrent graph captures corrupt CUDA context
// -- Multi-GPU pool data ---------------------------------------------------
// Per-slot descriptor. Slot engines are separate Engine<T> instances;
// they are never pooled recursively (their own m_slots deques are empty).
//
// Lifecycle in elastic mode:
// alive (engine != nullptr, busy == false) -- idle, ready for inference
// alive (engine != nullptr, busy == true) -- running inference
// dead (engine == nullptr) -- released, VRAM freed
// Dead entries stay in the deque to avoid invalidating pointers
// held by other threads. tryGrowPool() reuses them.
struct InferenceSlot {
int deviceIndex = 0;
bool busy = false;
size_t memUsed = 0; ///< VRAM bytes this slot holds
std::unique_ptr<Engine<T>> engine;
std::chrono::steady_clock::time_point lastUsedTime
= std::chrono::steady_clock::now(); ///< For idle-release
};
// Internal helpers -- implemented in EngineMultiGpu.inl
bool loadSlots(const ANSCENTER::Options& baseOptions,
const std::string& modelPath,
const std::array<float, 3>& subVals,
const std::array<float, 3>& divVals,
bool normalize,
bool fromOnnx,
int maxSlotsPerGpu,
double memSafetyFactor);
bool runInferenceFromPool(const std::vector<std::vector<cv::cuda::GpuMat>>& inputs,
std::vector<std::vector<std::vector<T>>>& featureVectors);
InferenceSlot* tryGrowPool(bool bypassGrace = false);
// -- Elastic pool storage ----------------------------------------------
//
// std::deque is used instead of std::vector because push_back on a deque
// does NOT invalidate references to existing elements. This is critical
// because runInferenceFromPool holds a raw InferenceSlot* across the GPU
// call while another thread may call tryGrowPool -> push_back concurrently.
std::deque<InferenceSlot> m_slots; ///< All managed Engine<T> instances
std::vector<GpuDeviceInfo> m_deviceInfos; ///< Per-device info from init
mutable std::mutex m_slotMutex; ///< Protects m_slots busy flags
std::condition_variable m_slotFreeCv; ///< Notified when any slot becomes idle
std::mutex m_growMutex; ///< Serialises slot creation
std::atomic<int> m_activeCount{0}; ///< Running inference count
std::atomic<int> m_totalCapacity{0}; ///< Alive slot count (atomic -- read lock-free by public API)
std::atomic<size_t> m_nextSlotHint{0}; ///< Round-robin start index for slot scan
// -- Elastic pool config (stored from loadSlots for on-demand growth) --
size_t m_memPerSlot{0};
std::string m_poolModelPath;
std::array<float, 3> m_poolSubVals{};
std::array<float, 3> m_poolDivVals{};
bool m_poolNormalize{true};
bool m_poolFromOnnx{false};
double m_poolSafetyFactor{0.80};
bool m_elasticMode{false}; ///< true when maxSlotsPerGpu == -1
// -- Global elastic slot cap (shared across ALL pools on ALL GPUs) --
// Prevents CUDA driver SRW lock convoy when too many concurrent contexts exist.
// Budget: ~4 slots per GB of total GPU VRAM (computed once on first pool init).
// 4 GB → 12, 6 GB → 24, 8 GB → 32, 12 GB → 48, 24 GB → 96
static std::atomic<int> s_globalElasticCount; ///< Current total elastic slots
static std::atomic<int> s_globalElasticMax; ///< Max total slots
static std::once_flag s_globalCapInitFlag; ///< Ensures max is computed once
// -- Elastic growth grace period --
// Delays elastic growth for N seconds after the most recent pool creation,
// giving later models time to create their probe engines before elastic
// growth from earlier pools consumes all free VRAM.
static constexpr int s_elasticGraceSec{5}; ///< Grace period in seconds (reduced from 30s: shared pools need fewer probes)
static std::atomic<int64_t> s_lastPoolCreatedMs; ///< Timestamp of last pool creation (ms since epoch)
// -- Automatic idle-slot cleanup timer (elastic mode only) --
std::thread m_idleTimerThread;
std::condition_variable m_idleTimerCv;
std::mutex m_idleTimerMutex;
std::atomic<bool> m_idleTimerStop{false};
double m_idleTimerIntervalSec{60.0}; ///< How often to check
double m_idleTimerThresholdSec{600.0}; ///< Idle time (seconds) before releasing — 10 minutes to avoid churn
void startIdleTimer();
void stopIdleTimer();
};
template <typename T> Engine<T>::Engine(const ANSCENTER::Options &options) : m_options(options) {
NvDynLoader::Initialize(); // no-op after first call; safe to call repeatedly
m_streamInitialized = false;
m_inferenceStream = nullptr;
m_lastBatchSize = -1;
m_inputBatchSize = 0;
m_memoryStream = nullptr;
m_firstInferenceCall = true;
m_batchShapeChangeLogged = false;
m_hasDynamicSpatialDims = false;
m_profileInitialized = false;
m_lastSetInputDims.clear();
m_preprocessedInputs.clear();
}
template <typename T> Engine<T>::Engine() {
NvDynLoader::Initialize();
m_streamInitialized = false;
m_inferenceStream = nullptr;
m_lastBatchSize = -1;
m_inputBatchSize = 0;
m_memoryStream = nullptr;
m_normalize = true;
m_firstInferenceCall = true;
m_batchShapeChangeLogged = false;
m_hasDynamicSpatialDims = false;
m_profileInitialized = false;
m_lastSetInputDims.clear();
}
template <typename T> Engine<T>::~Engine() {
// =========================================================================
// CRITICAL: Destructors must NEVER throw. Every CUDA/TRT call below is
// wrapped in try/catch so that a failure in one step does not skip the
// remaining cleanup (especially m_engine / m_runtime reset, which release
// TensorRT file handles and VRAM).
//
// Previous bug: clearGpuBuffers() called Util::checkCudaErrorCode() which
// throws on cudaFree failure. That aborted the destructor mid-way,
// leaving ICudaEngine/IRuntime alive → model folder locked on disk.
// =========================================================================
// Stop the idle-slot cleanup timer (if running) before touching pool state
try { stopIdleTimer(); } catch (...) {}
// Destroy cached CUDA graphs
try {
for (auto& [bs, ge] : m_graphExecs) { if (ge) cudaGraphExecDestroy(ge); }
m_graphExecs.clear();
} catch (...) {}
// Free pinned output buffers
try {
for (T* p : m_pinnedOutputBuffers) { if (p) cudaFreeHost(p); }
m_pinnedOutputBuffers.clear();
} catch (...) {}
// Free GPU I/O buffers — use cudaFree directly (NOT checkCudaErrorCode which throws)
try {
if (!m_buffers.empty()) {
for (void* ptr : m_buffers) {
if (ptr) cudaFree(ptr); // ignore errors — destructor must not throw
}
m_buffers.clear();
}
} catch (...) {}
try {
if (m_cudaStream) {
cudaStreamDestroy(m_cudaStream);
m_cudaStream = nullptr;
}
} catch (...) {}
try {
if (m_streamInitialized && m_inferenceStream) {
cudaStreamDestroy(m_inferenceStream);
m_inferenceStream = nullptr;
m_streamInitialized = false;
}
} catch (...) {}
try {
if (m_memoryStream) {
cudaStreamDestroy(m_memoryStream);
m_memoryStream = nullptr;
}
} catch (...) {}
// Release GPU clock lock if we hold one
try { unlockGpuClocks(); } catch (...) {}
// TRT destruction ordering: context MUST be destroyed before engine.
// With shared_ptr, m_engine may outlive this instance (other tasks share it).
// Explicit reset ensures correct order.
try { m_context.reset(); } catch (...) {}
// Release from TRTEngineCache (decrements refcount; engine freed when last user releases)
try {
if (m_usingCachedEngine && !m_cachedEnginePath.empty()) {
TRTEngineCache::instance().release(m_cachedEnginePath, m_cachedGpuIndex);
}
} catch (...) {}
// These MUST execute to release TensorRT file handles and VRAM.
// Previously, a throw in clearGpuBuffers() skipped these lines,
// leaving the model folder locked on disk.
try { m_engine.reset(); } catch (...) {}
try { m_runtime.reset(); } catch (...) {}
}
// ============================================================================
// NVML GPU Clock Lock — dynamically loaded, no build-time nvml.lib needed
// ============================================================================
// NVML type / constant definitions (subset — avoids #include <nvml.h>)
namespace nvml_types {
using Return_t = unsigned int; // nvmlReturn_t
using Device_t = void*; // nvmlDevice_t (opaque pointer)
enum : unsigned int { SUCCESS = 0 };
enum ClockType : unsigned int { CLOCK_GRAPHICS = 0 };
}
template <typename T>
void Engine<T>::lockGpuClocks(int deviceIndex, int requestedMHz) {
if (requestedMHz == 0) return; // disabled
#ifdef _WIN32
// Load nvml.dll from System32 (installed by NVIDIA driver)
auto hLib = LoadLibraryA("nvml.dll");
if (!hLib) {
if (m_verbose) std::cout << "Warning: nvml.dll not found — GPU clock lock unavailable" << std::endl;
return;
}
m_nvmlLib = static_cast<void*>(hLib);
// Resolve NVML function pointers
using fn_Init = nvml_types::Return_t(*)();
using fn_Shutdown = nvml_types::Return_t(*)();
using fn_GetHandle = nvml_types::Return_t(*)(unsigned int, nvml_types::Device_t*);
using fn_GetMaxClock = nvml_types::Return_t(*)(nvml_types::Device_t, nvml_types::ClockType, unsigned int*);
using fn_SetLocked = nvml_types::Return_t(*)(nvml_types::Device_t, unsigned int, unsigned int);
using fn_ErrorString = const char*(*)(nvml_types::Return_t);
auto pInit = reinterpret_cast<fn_Init>(GetProcAddress(hLib, "nvmlInit_v2"));
auto pShutdown = reinterpret_cast<fn_Shutdown>(GetProcAddress(hLib, "nvmlShutdown"));
auto pGetHandle = reinterpret_cast<fn_GetHandle>(GetProcAddress(hLib, "nvmlDeviceGetHandleByIndex_v2"));
auto pGetMaxClk = reinterpret_cast<fn_GetMaxClock>(GetProcAddress(hLib, "nvmlDeviceGetMaxClockInfo"));
auto pSetLocked = reinterpret_cast<fn_SetLocked>(GetProcAddress(hLib, "nvmlDeviceSetGpuLockedClocks"));
auto pErrStr = reinterpret_cast<fn_ErrorString>(GetProcAddress(hLib, "nvmlErrorString"));
if (!pInit || !pGetHandle || !pSetLocked) {
if (m_verbose) std::cout << "Warning: NVML symbols not found — GPU clock lock unavailable" << std::endl;
FreeLibrary(hLib); m_nvmlLib = nullptr;
return;
}
auto errName = [&](nvml_types::Return_t r) -> const char* {
return pErrStr ? pErrStr(r) : "unknown";
};
nvml_types::Return_t rc = pInit();
if (rc != nvml_types::SUCCESS) {
if (m_verbose) std::cout << "Warning: nvmlInit failed: " << errName(rc) << std::endl;
FreeLibrary(hLib); m_nvmlLib = nullptr;
return;
}
nvml_types::Device_t nvmlDev = nullptr;
rc = pGetHandle(static_cast<unsigned int>(deviceIndex), &nvmlDev);
if (rc != nvml_types::SUCCESS) {
if (m_verbose) std::cout << "Warning: nvmlDeviceGetHandleByIndex(" << deviceIndex << ") failed: " << errName(rc) << std::endl;
pShutdown(); FreeLibrary(hLib); m_nvmlLib = nullptr;
return;
}
unsigned int targetMHz = 0;
if (requestedMHz < 0) {
// Auto mode: query max graphics clock and use ~80%
unsigned int maxClock = 0;
if (pGetMaxClk) {
rc = pGetMaxClk(nvmlDev, nvml_types::CLOCK_GRAPHICS, &maxClock);
if (rc == nvml_types::SUCCESS && maxClock > 0) {
targetMHz = static_cast<unsigned int>(maxClock * 0.80);
targetMHz = (targetMHz / 15) * 15; // Round down to 15 MHz step
if (targetMHz < 210) targetMHz = 210;
} else {
if (m_verbose) std::cout << "Warning: Could not query max GPU clock — skipping clock lock" << std::endl;
pShutdown(); FreeLibrary(hLib); m_nvmlLib = nullptr;
return;
}
}
} else {
targetMHz = static_cast<unsigned int>(requestedMHz);
}
rc = pSetLocked(nvmlDev, targetMHz, targetMHz);
if (rc == nvml_types::SUCCESS) {
m_clocksLocked = true;
m_nvmlDeviceIdx = static_cast<unsigned int>(deviceIndex);
if (m_verbose) std::cout << "Info: GPU clocks locked at " << targetMHz << " MHz (device " << deviceIndex << ")" << std::endl;
} else {
if (m_verbose) {
std::cout << "Warning: nvmlDeviceSetGpuLockedClocks failed: " << errName(rc) << std::endl;
std::cout << " (Run as Administrator, or use: nvidia-smi -lgc " << targetMHz << "," << targetMHz << ")" << std::endl;
}
pShutdown(); FreeLibrary(hLib); m_nvmlLib = nullptr;
}
#else
// Linux: similar pattern with dlopen("libnvidia-ml.so.1", RTLD_LAZY)
// Left as TODO — nvidia-smi -lgc works without elevation on Linux.
if (m_verbose) std::cout << "Info: GPU clock lock not implemented on this platform — use nvidia-smi -lgc" << std::endl;
#endif
}
template <typename T>
void Engine<T>::unlockGpuClocks() {
if (!m_clocksLocked || !m_nvmlLib) return;
#ifdef _WIN32
auto hLib = static_cast<HMODULE>(m_nvmlLib);
using fn_GetHandle = nvml_types::Return_t(*)(unsigned int, nvml_types::Device_t*);
using fn_ResetClk = nvml_types::Return_t(*)(nvml_types::Device_t);
using fn_Shutdown = nvml_types::Return_t(*)();
auto pGetHandle = reinterpret_cast<fn_GetHandle>(GetProcAddress(hLib, "nvmlDeviceGetHandleByIndex_v2"));
auto pResetClk = reinterpret_cast<fn_ResetClk>(GetProcAddress(hLib, "nvmlDeviceResetGpuLockedClocks"));
auto pShutdown = reinterpret_cast<fn_Shutdown>(GetProcAddress(hLib, "nvmlShutdown"));
if (pGetHandle && pResetClk) {
nvml_types::Device_t nvmlDev = nullptr;
if (pGetHandle(m_nvmlDeviceIdx, &nvmlDev) == nvml_types::SUCCESS) {
pResetClk(nvmlDev);
if (m_verbose) std::cout << "Info: GPU clocks unlocked (device " << m_nvmlDeviceIdx << ")" << std::endl;
}
}
if (pShutdown) pShutdown();
FreeLibrary(hLib);
#endif
m_nvmlLib = nullptr;
m_clocksLocked = false;
}
namespace Util {
inline bool doesFileExist(const std::string& filepath) {
std::ifstream f(filepath.c_str());
return f.good();
}
inline void checkCudaErrorCode(cudaError_t code) {
if (code != cudaSuccess) {
std::string errMsg = "CUDA operation failed with code: " + std::to_string(code) + " (" + cudaGetErrorName(code) +
"), with message: " + cudaGetErrorString(code);
std::cout<<errMsg;
}
}
inline std::vector<std::string> getFilesInDirectory(const std::string& dirPath) {
std::vector<std::string> fileNames;
for (const auto& entry : std::filesystem::directory_iterator(dirPath)) {
if (entry.is_regular_file()) {
fileNames.push_back(entry.path().string());
}
}
return fileNames;
}
}
#include "engine/EngineRunInference.inl"
#include "engine/EngineUtilities.inl"
#include "engine/EngineBuildLoadNetwork.inl"
#include "engine/EngineMultiGpu.inl"
// ============================================================================
// Explicit template instantiation declarations -- DLL consumers only
//
// These tell each consuming translation unit: "Engine<float> is already fully
// compiled and exported by the DLL. Do NOT re-instantiate it here; resolve
// the symbols at link time from the DLL's import library instead."
//
// This suppresses redundant code generation in every consuming TU and
// guarantees that only ONE definition of each Engine<T> specialisation exists
// across the whole program (in the DLL).
//
// To add a new exported instantiation (e.g. Engine<uint8_t>):
// 1. Add a matching template class ENGINE_API Engine<uint8_t>; in engine.cpp
// 2. Add the corresponding extern declaration below.
// ============================================================================
#ifndef ENGINE_EXPORTS
extern template class ENGINE_API Engine<float>;
#endif
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
#define NOMINMAX // Prevent windows.h from defining min/max macros
// which break std::min / std::max (C2589)
// Windows Header Files
#include <windows.h>
File diff suppressed because it is too large Load Diff
+749
View File
@@ -0,0 +1,749 @@
// nv12_to_rgb.cu — CUDA kernels for NV12→RGB and NV12→BGR conversion.
//
// Two use cases:
// 1. launchNV12ToRGB() — inference fast path (ANSRTYOLO uploads NV12 directly)
// 2. ANSGpuNV12ToBGR() — video_player display path (replaces slow CPU cvtColorTwoPlane)
//
// cv::cuda::cvtColor does NOT support NV12, so we use custom kernels.
#include <cuda_runtime.h>
#include <cstdint>
#include <cstdio>
// ── Shared YUV→RGB computation ───────────────────────────────────────────
__device__ __forceinline__ void nv12_yuv_to_rgb(
float Y, float U, float V,
uint8_t& r, uint8_t& g, uint8_t& b)
{
// BT.601 conversion (same as FFmpeg swscale / OpenCV cvtColorTwoPlane)
const float R = Y + 1.402f * V;
const float G = Y - 0.344136f * U - 0.714136f * V;
const float B = Y + 1.772f * U;
r = static_cast<uint8_t>(fminf(fmaxf(R, 0.f), 255.f));
g = static_cast<uint8_t>(fminf(fmaxf(G, 0.f), 255.f));
b = static_cast<uint8_t>(fminf(fmaxf(B, 0.f), 255.f));
}
__device__ __forceinline__ void nv12_read_yuv(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
int x, int y,
float& Y, float& U, float& V)
{
Y = static_cast<float>(yPlane[y * yPitch + x]);
const int uvIdx = (y >> 1) * uvPitch + (x & ~1);
U = static_cast<float>(uvPlane[uvIdx]) - 128.f;
V = static_cast<float>(uvPlane[uvIdx + 1]) - 128.f;
}
// ── NV12→RGB kernel (for inference fast path) ────────────────────────────
__global__ void nv12ToRgbKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ rgbOut, int outPitch,
int width, int height)
{
const int x = blockIdx.x * blockDim.x + threadIdx.x;
const int y = blockIdx.y * blockDim.y + threadIdx.y;
if (x >= width || y >= height) return;
float Y, U, V;
nv12_read_yuv(yPlane, yPitch, uvPlane, uvPitch, x, y, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
const int outIdx = y * outPitch + x * 3;
rgbOut[outIdx] = r;
rgbOut[outIdx + 1] = g;
rgbOut[outIdx + 2] = b;
}
// ── NV12→BGR kernel (for LabVIEW display — matches OpenCV channel order) ─
__global__ void nv12ToBgrKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ bgrOut, int outPitch,
int width, int height)
{
const int x = blockIdx.x * blockDim.x + threadIdx.x;
const int y = blockIdx.y * blockDim.y + threadIdx.y;
if (x >= width || y >= height) return;
float Y, U, V;
nv12_read_yuv(yPlane, yPitch, uvPlane, uvPitch, x, y, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
const int outIdx = y * outPitch + x * 3;
bgrOut[outIdx] = b; // BGR order for OpenCV
bgrOut[outIdx + 1] = g;
bgrOut[outIdx + 2] = r;
}
// ── Fused NV12→RGB + letterbox resize kernel ────────────────────────────
// Runs at OUTPUT resolution (e.g. 640×640), sampling from 4K NV12 source
// with bilinear interpolation. This avoids the 24MB 4K RGB intermediate
// and processes 20× fewer pixels than separate convert + resize.
__device__ __forceinline__ void nv12_read_yuv_bilinear(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
float sx, float sy, int srcW, int srcH,
float& Y, float& U, float& V)
{
// Bilinear interpolation for Y plane
int x0 = static_cast<int>(sx);
int y0 = static_cast<int>(sy);
int x1 = min(x0 + 1, srcW - 1);
int y1 = min(y0 + 1, srcH - 1);
x0 = max(x0, 0);
y0 = max(y0, 0);
float fx = sx - static_cast<float>(x0);
float fy = sy - static_cast<float>(y0);
// Bilinear Y
float y00 = static_cast<float>(yPlane[y0 * yPitch + x0]);
float y01 = static_cast<float>(yPlane[y0 * yPitch + x1]);
float y10 = static_cast<float>(yPlane[y1 * yPitch + x0]);
float y11 = static_cast<float>(yPlane[y1 * yPitch + x1]);
Y = y00 * (1.f - fx) * (1.f - fy) + y01 * fx * (1.f - fy)
+ y10 * (1.f - fx) * fy + y11 * fx * fy;
// UV: NV12 has interleaved U/V at half resolution
// Sample at corresponding chroma position
float csx = sx * 0.5f;
float csy = sy * 0.5f;
int cx0 = static_cast<int>(csx);
int cy0 = static_cast<int>(csy);
int cx1 = min(cx0 + 1, srcW / 2 - 1);
int cy1 = min(cy0 + 1, srcH / 2 - 1);
cx0 = max(cx0, 0);
cy0 = max(cy0, 0);
float cfx = csx - static_cast<float>(cx0);
float cfy = csy - static_cast<float>(cy0);
// Bilinear U
float u00 = static_cast<float>(uvPlane[cy0 * uvPitch + cx0 * 2]);
float u01 = static_cast<float>(uvPlane[cy0 * uvPitch + cx1 * 2]);
float u10 = static_cast<float>(uvPlane[cy1 * uvPitch + cx0 * 2]);
float u11 = static_cast<float>(uvPlane[cy1 * uvPitch + cx1 * 2]);
U = (u00 * (1.f - cfx) * (1.f - cfy) + u01 * cfx * (1.f - cfy)
+ u10 * (1.f - cfx) * cfy + u11 * cfx * cfy) - 128.f;
// Bilinear V
float v00 = static_cast<float>(uvPlane[cy0 * uvPitch + cx0 * 2 + 1]);
float v01 = static_cast<float>(uvPlane[cy0 * uvPitch + cx1 * 2 + 1]);
float v10 = static_cast<float>(uvPlane[cy1 * uvPitch + cx0 * 2 + 1]);
float v11 = static_cast<float>(uvPlane[cy1 * uvPitch + cx1 * 2 + 1]);
V = (v00 * (1.f - cfx) * (1.f - cfy) + v01 * cfx * (1.f - cfy)
+ v10 * (1.f - cfx) * cfy + v11 * cfx * cfy) - 128.f;
}
__global__ void nv12ToRgbLetterboxKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ rgbOut, int outPitch,
int outW, int outH, // model input size (e.g. 640×640)
int srcW, int srcH, // NV12 source size (e.g. 3840×2160)
int unpadW, int unpadH, // letterboxed content size within output
float invScale) // 1/scale = srcDim/unpadDim
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
const int outIdx = oy * outPitch + ox * 3;
// Letterbox padding: black for pixels outside content area
if (ox >= unpadW || oy >= unpadH) {
rgbOut[outIdx] = 0;
rgbOut[outIdx + 1] = 0;
rgbOut[outIdx + 2] = 0;
return;
}
// Map output pixel to source NV12 coordinates
float sx = (static_cast<float>(ox) + 0.5f) * invScale - 0.5f;
float sy = (static_cast<float>(oy) + 0.5f) * invScale - 0.5f;
sx = fmaxf(sx, 0.f);
sy = fmaxf(sy, 0.f);
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
rgbOut[outIdx] = r;
rgbOut[outIdx + 1] = g;
rgbOut[outIdx + 2] = b;
}
// ── Host-callable wrappers ───────────────────────────────────────────────
extern "C" void launchNV12ToRGBLetterbox(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
int unpadW, int unpadH,
float invScale,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12ToRgbLetterboxKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
rgbOut, outPitch, outW, outH,
srcW, srcH, unpadW, unpadH, invScale);
}
extern "C" void launchNV12ToRGB(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int width, int height,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((width + block.x - 1) / block.x,
(height + block.y - 1) / block.y);
nv12ToRgbKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
rgbOut, outPitch, width, height);
}
// ── Fused NV12→RGB + direct resize kernel (for classification) ───────────
// Like letterbox kernel but stretches to fill the entire output (no padding).
// Used for classification models that expect direct resize, not aspect-
// preserving letterbox.
__global__ void nv12ToRgbResizeKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ rgbOut, int outPitch,
int outW, int outH, // model input size (e.g. 224×224)
int srcW, int srcH) // NV12 source size (e.g. 2688×1520)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
// Map output pixel to source NV12 coordinates (stretch to fill)
float sx = (static_cast<float>(ox) + 0.5f) * static_cast<float>(srcW) / static_cast<float>(outW) - 0.5f;
float sy = (static_cast<float>(oy) + 0.5f) * static_cast<float>(srcH) / static_cast<float>(outH) - 0.5f;
sx = fmaxf(sx, 0.f);
sy = fmaxf(sy, 0.f);
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
const int outIdx = oy * outPitch + ox * 3;
rgbOut[outIdx] = r;
rgbOut[outIdx + 1] = g;
rgbOut[outIdx + 2] = b;
}
extern "C" void launchNV12ToRGBResize(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12ToRgbResizeKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
rgbOut, outPitch, outW, outH,
srcW, srcH);
}
// ── Fused NV12→RGB + resize + CHW planar output (for SAM3 image encoder) ─
// Writes R, G, B to separate contiguous planes: [R plane][G plane][B plane]
// at model input resolution. Eliminates cv::resize + cvtColor + split + memcpy.
__global__ void nv12ToRgbResizeCHW_uint8Kernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ chwOut, // contiguous CHW: R plane, G plane, B plane
int outW, int outH,
int srcW, int srcH)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
float sx = (static_cast<float>(ox) + 0.5f) * static_cast<float>(srcW) / static_cast<float>(outW) - 0.5f;
float sy = (static_cast<float>(oy) + 0.5f) * static_cast<float>(srcH) / static_cast<float>(outH) - 0.5f;
sx = fmaxf(sx, 0.f);
sy = fmaxf(sy, 0.f);
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
const int planeSize = outW * outH;
const int idx = oy * outW + ox;
chwOut[idx] = r; // R plane
chwOut[idx + planeSize] = g; // G plane
chwOut[idx + 2 * planeSize] = b; // B plane
}
__global__ void nv12ToRgbResizeCHW_floatKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
float* __restrict__ chwOut, // contiguous CHW float32 [0, 255]
int outW, int outH,
int srcW, int srcH)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
float sx = (static_cast<float>(ox) + 0.5f) * static_cast<float>(srcW) / static_cast<float>(outW) - 0.5f;
float sy = (static_cast<float>(oy) + 0.5f) * static_cast<float>(srcH) / static_cast<float>(outH) - 0.5f;
sx = fmaxf(sx, 0.f);
sy = fmaxf(sy, 0.f);
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
const int planeSize = outW * outH;
const int idx = oy * outW + ox;
chwOut[idx] = static_cast<float>(r);
chwOut[idx + planeSize] = static_cast<float>(g);
chwOut[idx + 2 * planeSize] = static_cast<float>(b);
}
extern "C" void launchNV12ToRGBResizeCHW_uint8(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* chwOut,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12ToRgbResizeCHW_uint8Kernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
chwOut, outW, outH, srcW, srcH);
}
extern "C" void launchNV12ToRGBResizeCHW_float(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
float* chwOut,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12ToRgbResizeCHW_floatKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
chwOut, outW, outH, srcW, srcH);
}
extern "C" void launchNV12ToBGR(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int width, int height,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((width + block.x - 1) / block.x,
(height + block.y - 1) / block.y);
nv12ToBgrKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
bgrOut, outPitch, width, height);
}
// ── Fused NV12→BGR + bilinear resize kernel (for OCR detection) ──────────
// Same algorithm as nv12ToRgbResizeKernel but outputs BGR channel order.
__global__ void nv12ToBgrResizeKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ bgrOut, int outPitch,
int outW, int outH,
int srcW, int srcH)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
float sx = (static_cast<float>(ox) + 0.5f) * static_cast<float>(srcW) / static_cast<float>(outW) - 0.5f;
float sy = (static_cast<float>(oy) + 0.5f) * static_cast<float>(srcH) / static_cast<float>(outH) - 0.5f;
sx = fmaxf(sx, 0.f);
sy = fmaxf(sy, 0.f);
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
const int outIdx = oy * outPitch + ox * 3;
bgrOut[outIdx] = b;
bgrOut[outIdx + 1] = g;
bgrOut[outIdx + 2] = r;
}
extern "C" void launchNV12ToBGRResize(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12ToBgrResizeKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
bgrOut, outPitch, outW, outH, srcW, srcH);
}
// ── Center-padded NV12→RGB + letterbox resize kernel (for SCRFD) ─────────
// Same as nv12ToRgbLetterboxKernel but content is centered with (padLeft, padTop)
// offset instead of placed at top-left origin. Used by SCRFD face detector
// which uses center-padding (dw = pad_w/2, dh = pad_h/2).
__global__ void nv12ToRgbCenterLetterboxKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
int unpadW, int unpadH,
float invScale,
int padLeft, int padTop)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
const int outIdx = oy * outPitch + ox * 3;
// Center-padding: black for pixels outside the content area
const int cx = ox - padLeft;
const int cy = oy - padTop;
if (cx < 0 || cx >= unpadW || cy < 0 || cy >= unpadH) {
rgbOut[outIdx] = 0;
rgbOut[outIdx + 1] = 0;
rgbOut[outIdx + 2] = 0;
return;
}
// Map content pixel to source NV12 coordinates
float sx = (static_cast<float>(cx) + 0.5f) * invScale - 0.5f;
float sy = (static_cast<float>(cy) + 0.5f) * invScale - 0.5f;
sx = fmaxf(sx, 0.f);
sy = fmaxf(sy, 0.f);
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
rgbOut[outIdx] = r;
rgbOut[outIdx + 1] = g;
rgbOut[outIdx + 2] = b;
}
extern "C" void launchNV12ToRGBCenterLetterbox(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* rgbOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
int unpadW, int unpadH,
float invScale,
int padLeft, int padTop,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12ToRgbCenterLetterboxKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
rgbOut, outPitch, outW, outH,
srcW, srcH, unpadW, unpadH, invScale,
padLeft, padTop);
}
// ── NV12 affine warp → BGR kernel (for face alignment) ──────────────────
// Reads from full-resolution NV12 source, applies inverse affine transform,
// and outputs a small aligned face image (e.g. 112×112) in BGR order.
// The inverse affine maps each output pixel back to the NV12 source coords.
// Border handling: replicate edge (matches cv::BORDER_REPLICATE).
// Outputs BGR to match existing pipeline (ANSFaceRecognizer expects BGR).
__global__ void nv12AffineWarpToBgrKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ bgrOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
float m00, float m01, float m02,
float m10, float m11, float m12)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
// Inverse affine: map output pixel to source NV12 coordinate
float sx = m00 * ox + m01 * oy + m02;
float sy = m10 * ox + m11 * oy + m12;
// Border replicate: clamp to source bounds
sx = fmaxf(0.f, fminf(sx, static_cast<float>(srcW - 1)));
sy = fmaxf(0.f, fminf(sy, static_cast<float>(srcH - 1)));
float Y, U, V;
nv12_read_yuv_bilinear(yPlane, yPitch, uvPlane, uvPitch,
sx, sy, srcW, srcH, Y, U, V);
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
// Output BGR order (matches OpenCV convention for downstream pipeline)
const int outIdx = oy * outPitch + ox * 3;
bgrOut[outIdx] = b;
bgrOut[outIdx + 1] = g;
bgrOut[outIdx + 2] = r;
}
extern "C" void launchNV12AffineWarpToBGR(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH,
int srcW, int srcH,
float m00, float m01, float m02,
float m10, float m11, float m12,
cudaStream_t stream)
{
dim3 block(16, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12AffineWarpToBgrKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
bgrOut, outPitch, outW, outH,
srcW, srcH,
m00, m01, m02, m10, m11, m12);
}
// ── High-level NV12→BGR: CPU→GPU upload, kernel, GPU→CPU download ────────
// Thread-local GPU buffers avoid per-call cudaMalloc overhead.
// Exported for cross-DLL use by video_player via GetProcAddress.
struct GpuNV12Buffers {
uint8_t* d_yPlane = nullptr;
uint8_t* d_uvPlane = nullptr;
uint8_t* d_bgrOut = nullptr;
int allocWidth = 0;
int allocHeight = 0;
cudaStream_t stream = nullptr;
void ensureAllocated(int w, int h) {
if (w == allocWidth && h == allocHeight && d_yPlane) return;
free();
size_t ySize = (size_t)w * h;
size_t uvSize = (size_t)w * (h / 2);
size_t bgrSize = (size_t)w * h * 3;
cudaMalloc(&d_yPlane, ySize);
cudaMalloc(&d_uvPlane, uvSize);
cudaMalloc(&d_bgrOut, bgrSize);
cudaStreamCreate(&stream);
allocWidth = w;
allocHeight = h;
}
void free() {
if (d_yPlane) { cudaFree(d_yPlane); d_yPlane = nullptr; }
if (d_uvPlane) { cudaFree(d_uvPlane); d_uvPlane = nullptr; }
if (d_bgrOut) { cudaFree(d_bgrOut); d_bgrOut = nullptr; }
if (stream) { cudaStreamDestroy(stream); stream = nullptr; }
allocWidth = allocHeight = 0;
}
~GpuNV12Buffers() { free(); }
};
static thread_local GpuNV12Buffers t_bufs;
extern "C" __declspec(dllexport)
int ANSGpuNV12ToBGR(
const uint8_t* yPlane, int yLinesize,
const uint8_t* uvPlane, int uvLinesize,
uint8_t* bgrOut, int bgrStep,
int width, int height, int gpuIndex)
{
if (!yPlane || !uvPlane || !bgrOut || width <= 0 || height <= 0) return -1;
// Select GPU
if (gpuIndex >= 0) {
cudaError_t err = cudaSetDevice(gpuIndex);
if (err != cudaSuccess) {
fprintf(stderr, "ANSGpuNV12ToBGR: cudaSetDevice(%d) failed: %s\n",
gpuIndex, cudaGetErrorString(err));
return -2;
}
}
t_bufs.ensureAllocated(width, height);
if (!t_bufs.d_yPlane) return -3; // allocation failed
// Upload Y and UV planes to GPU
cudaMemcpy2DAsync(t_bufs.d_yPlane, width,
yPlane, yLinesize,
width, height,
cudaMemcpyHostToDevice, t_bufs.stream);
cudaMemcpy2DAsync(t_bufs.d_uvPlane, width,
uvPlane, uvLinesize,
width, height / 2,
cudaMemcpyHostToDevice, t_bufs.stream);
// Launch BGR kernel
dim3 block(32, 8);
dim3 grid((width + block.x - 1) / block.x,
(height + block.y - 1) / block.y);
nv12ToBgrKernel<<<grid, block, 0, t_bufs.stream>>>(
t_bufs.d_yPlane, width,
t_bufs.d_uvPlane, width,
t_bufs.d_bgrOut, width * 3,
width, height);
// Download BGR result back to CPU
cudaMemcpy2DAsync(bgrOut, bgrStep,
t_bufs.d_bgrOut, width * 3,
width * 3, height,
cudaMemcpyDeviceToHost, t_bufs.stream);
cudaStreamSynchronize(t_bufs.stream);
// Check for errors
cudaError_t err = cudaGetLastError();
if (err != cudaSuccess) {
fprintf(stderr, "ANSGpuNV12ToBGR: CUDA error: %s\n", cudaGetErrorString(err));
return -4;
}
return 0; // success
}
// ============================================================================
// NV12 rectangular crop → BGR kernel
// Reads a rectangular region from the full NV12 frame and outputs BGR.
// No resize — output is the same size as the crop region.
// ============================================================================
__global__ void nv12CropToBGRKernel(
const uint8_t* __restrict__ yPlane, int yPitch,
const uint8_t* __restrict__ uvPlane, int uvPitch,
uint8_t* __restrict__ bgrOut, int outPitch,
int outW, int outH,
int cropX, int cropY,
int srcW, int srcH)
{
const int ox = blockIdx.x * blockDim.x + threadIdx.x;
const int oy = blockIdx.y * blockDim.y + threadIdx.y;
if (ox >= outW || oy >= outH) return;
// Map output pixel to source coordinate
const int sx = cropX + ox;
const int sy = cropY + oy;
// Clamp to source bounds
const int cx = min(max(sx, 0), srcW - 1);
const int cy = min(max(sy, 0), srcH - 1);
// Read Y
const float Y = static_cast<float>(yPlane[cy * yPitch + cx]);
// Bilinear UV (chroma subsampled 2:1 in both dimensions, interleaved NV12)
// Chroma sample centers are at (0.5, 0.5) offset in luma coords, so map
// luma (cx,cy) → chroma continuous coords, then bilinear-interpolate.
const float csx = static_cast<float>(cx) * 0.5f;
const float csy = static_cast<float>(cy) * 0.5f;
const int cx0 = static_cast<int>(csx);
const int cy0 = static_cast<int>(csy);
const int cx1 = min(cx0 + 1, (srcW >> 1) - 1);
const int cy1 = min(cy0 + 1, (srcH >> 1) - 1);
const float cfx = csx - static_cast<float>(cx0);
const float cfy = csy - static_cast<float>(cy0);
const float u00 = static_cast<float>(uvPlane[cy0 * uvPitch + cx0 * 2]);
const float u01 = static_cast<float>(uvPlane[cy0 * uvPitch + cx1 * 2]);
const float u10 = static_cast<float>(uvPlane[cy1 * uvPitch + cx0 * 2]);
const float u11 = static_cast<float>(uvPlane[cy1 * uvPitch + cx1 * 2]);
const float U = (u00 * (1.f - cfx) * (1.f - cfy) + u01 * cfx * (1.f - cfy)
+ u10 * (1.f - cfx) * cfy + u11 * cfx * cfy) - 128.f;
const float v00 = static_cast<float>(uvPlane[cy0 * uvPitch + cx0 * 2 + 1]);
const float v01 = static_cast<float>(uvPlane[cy0 * uvPitch + cx1 * 2 + 1]);
const float v10 = static_cast<float>(uvPlane[cy1 * uvPitch + cx0 * 2 + 1]);
const float v11 = static_cast<float>(uvPlane[cy1 * uvPitch + cx1 * 2 + 1]);
const float V = (v00 * (1.f - cfx) * (1.f - cfy) + v01 * cfx * (1.f - cfy)
+ v10 * (1.f - cfx) * cfy + v11 * cfx * cfy) - 128.f;
// BT.601 YUV → RGB
uint8_t r, g, b;
nv12_yuv_to_rgb(Y, U, V, r, g, b);
// Write BGR (OpenCV channel order)
const int outIdx = oy * outPitch + ox * 3;
bgrOut[outIdx] = b;
bgrOut[outIdx + 1] = g;
bgrOut[outIdx + 2] = r;
}
extern "C" void launchNV12CropToBGR(
const uint8_t* yPlane, int yPitch,
const uint8_t* uvPlane, int uvPitch,
uint8_t* bgrOut, int outPitch,
int outW, int outH,
int cropX, int cropY,
int srcW, int srcH,
cudaStream_t stream)
{
dim3 block(32, 8);
dim3 grid((outW + block.x - 1) / block.x,
(outH + block.y - 1) / block.y);
nv12CropToBGRKernel<<<grid, block, 0, stream>>>(
yPlane, yPitch, uvPlane, uvPitch,
bgrOut, outPitch,
outW, outH,
cropX, cropY,
srcW, srcH);
}
+704
View File
@@ -0,0 +1,704 @@
import re, sys
CPP = r'C:\Projects\ANLS\ANSLIB\ANSODEngine\ANSTENSORRTCL.cpp'
with open(CPP, 'rb') as f:
data = f.read()
print('File size:', len(data))
CRLF = b'\r\n' in data
print('CRLF:', CRLF)
errors = []
def replace_once(data, old, new, label):
count = data.count(old)
if count != 1:
errors.append(f'{label}: found {count}, expected 1')
return data
return data.replace(old, new)
def replace_all(data, old, new, expected, label):
count = data.count(old)
if count != expected:
errors.append(f'{label}: found {count}, expected {expected}')
return data
return data.replace(old, new)
def replace_slice(data, start_anchor, end_anchor, new_content, label):
si = data.find(start_anchor)
if si == -1:
errors.append(f'{label}: start anchor not found: {start_anchor[:60]}')
return data
ei = data.find(end_anchor, si + len(start_anchor))
if ei == -1:
errors.append(f'{label}: end anchor not found: {end_anchor[:60]}')
return data
print(f' {label}: [{si}:{ei}] ({ei-si} bytes) -> {len(new_content)} bytes')
return data[:si] + new_content + data[ei:]
# ──────────────────────────────────────────────────────────────────────────────
# Step 1: Add #include <future>
# ──────────────────────────────────────────────────────────────────────────────
data = replace_once(data,
b'#include <opencv2/cudaimgproc.hpp>\r\n',
b'#include <opencv2/cudaimgproc.hpp>\r\n#include <future>\r\n',
'Step 1: Add #include <future>')
print('Step 1: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 2: buildLoadNetwork — add m_maxSlotsPerGpu (3 calls)
# ──────────────────────────────────────────────────────────────────────────────
data = replace_all(data,
b'buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE);',
b'buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);',
3, 'Step 2: buildLoadNetwork add m_maxSlotsPerGpu')
print('Step 2: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 2b: Remove _isFixedBatch = false; from LoadModel and LoadModelFromFolder
# Use surrounding context to avoid matching inside 3-tab Initialize line
# ──────────────────────────────────────────────────────────────────────────────
data = replace_all(data,
b'\t\t_isFixedBatch = false;\r\n\t\tstd::lock_guard',
b'\t\tstd::lock_guard',
2, 'Step 2b: Remove _isFixedBatch=false in LoadModel/LoadModelFromFolder')
print('Step 2b: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 2c: Remove _isFixedBatch = false; from Initialize (3-tab indent)
# ──────────────────────────────────────────────────────────────────────────────
data = replace_all(data,
b'\t\t\t_isFixedBatch = false;\r\n',
b'',
1, 'Step 2c: Remove _isFixedBatch=false in Initialize')
print('Step 2c: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 3: RunInference(camera_id) — release mutex before DetectObjects
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)\r\n',
b'\tstd::vector<std::vector<Object>> TENSORRTCL::RunInferencesBatch(',
b'TENSORRTCL::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)\r\n'
b'\t{\r\n'
b'\t\t// Validate state under brief lock\r\n'
b'\t\t{\r\n'
b'\t\t\tstd::lock_guard<std::recursive_mutex> lock(_mutex);\r\n'
b'\t\t\tif (!_modelLoadValid) {\r\n'
b'\t\t\t\t_logger.LogError("TENSORRTCL::RunInference",\r\n'
b'\t\t\t\t\t"Cannot load the TensorRT model. Please check if it exists",\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (!_licenseValid) {\r\n'
b'\t\t\t\t_logger.LogError("TENSORRTCL::RunInference",\r\n'
b'\t\t\t\t\t"Runtime license is not valid or expired. Please contact ANSCENTER",\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (!_isInitialized) {\r\n'
b'\t\t\t\t_logger.LogError("TENSORRTCL::RunInference",\r\n'
b'\t\t\t\t\t"Model is not initialized",\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (inputImgBGR.empty() || inputImgBGR.cols < 5 || inputImgBGR.rows < 5) {\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\t\t}\r\n'
b'\t\ttry {\r\n'
b'\t\t\treturn DetectObjects(inputImgBGR, camera_id);\r\n'
b'\t\t}\r\n'
b'\t\tcatch (const std::exception& e) {\r\n'
b'\t\t\t_logger.LogFatal("TENSORRTCL::RunInference", e.what(), __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\t}\r\n'
b'\t',
'Step 3: RunInference brief lock')
print('Step 3: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 4: RunInferencesBatch — brief lock, remove _isFixedBatch fallback
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::RunInferencesBatch(const std::vector<cv::Mat>& inputs, const std::string& camera_id) {\r\n',
b'\tTENSORRTCL::~TENSORRTCL() {',
b'TENSORRTCL::RunInferencesBatch(const std::vector<cv::Mat>& inputs, const std::string& camera_id) {\r\n'
b'\t\t// Validate state under brief lock\r\n'
b'\t\t{\r\n'
b'\t\t\tstd::lock_guard<std::recursive_mutex> lock(_mutex);\r\n'
b'\t\t\tif (!_modelLoadValid) {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::RunInferencesBatch",\r\n'
b'\t\t\t\t\t"Cannot load the TensorRT model. Please check if it exists", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (!_licenseValid) {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::RunInferencesBatch",\r\n'
b'\t\t\t\t\t"Runtime license is not valid or expired. Please contact ANSCENTER", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (!_isInitialized) {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::RunInferencesBatch",\r\n'
b'\t\t\t\t\t"Engine not initialized", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (inputs.empty()) return {};\r\n'
b'\t\t}\r\n'
b'\t\ttry {\r\n'
b'\t\t\treturn DetectObjectsBatch(inputs, camera_id);\r\n'
b'\t\t}\r\n'
b'\t\tcatch (const std::exception& e) {\r\n'
b'\t\t\tthis->_logger.LogFatal("TENSORRTCL::RunInferencesBatch", e.what(), __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\t};\r\n'
b'\t',
'Step 4: RunInferencesBatch brief lock')
print('Step 4: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 5: DetectObjects — three-phase mutex
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {\r\n',
b'\tstd::vector<std::vector<cv::cuda::GpuMat>> TENSORRTCL::Preprocess(',
b'TENSORRTCL::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {\r\n'
b'\t\t// Phase 1: Preprocess under brief lock\r\n'
b'\t\tImageMetadata meta;\r\n'
b'\t\tstd::vector<std::vector<cv::cuda::GpuMat>> input;\r\n'
b'\t\t{\r\n'
b'\t\t\tstd::lock_guard<std::recursive_mutex> lock(_mutex);\r\n'
b'\t\t\tinput = Preprocess(inputImage, meta);\r\n'
b'\t\t}\r\n'
b'\t\tif (input.empty()) return {};\r\n'
b'\r\n'
b'\t\t// Phase 2: Inference \xe2\x80\x94 mutex released; pool dispatches to idle GPU slot\r\n'
b'\t\tstd::vector<std::vector<std::vector<float>>> featureVectors;\r\n'
b'\t\tauto succ = m_trtEngine->runInference(input, featureVectors);\r\n'
b'\t\tif (!succ) {\r\n'
b'\t\t\tthis->_logger.LogFatal("TENSORRTCL::DetectObjects", "Error running inference", __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\r\n'
b'\t\t// Phase 3: Postprocess under brief lock\r\n'
b'\t\tstd::lock_guard<std::recursive_mutex> lock(_mutex);\r\n'
b'\t\tstd::vector<float> featureVector;\r\n'
b'\t\tEngine<float>::transformOutput(featureVectors, featureVector);\r\n'
b'\t\treturn Postprocess(featureVector, camera_id, meta);\r\n'
b'\t}\r\n'
b'\t',
'Step 5: DetectObjects three-phase mutex')
print('Step 5: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 6: Preprocess — add outMeta param, remove lock, replace member writes
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::Preprocess(const cv::Mat& inputImage) {\r\n',
b'\tstd::vector<Object> TENSORRTCL::Postprocess(',
b'TENSORRTCL::Preprocess(const cv::Mat& inputImage, ImageMetadata& outMeta) {\r\n'
b'\t\ttry {\r\n'
b'\t\t\tif (!_licenseValid) {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::Preprocess", "Invalid license", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif (inputImage.empty()) {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::Preprocess", "Input image is empty", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tif ((inputImage.cols < 5) || (inputImage.rows < 5)) {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::Preprocess",\r\n'
b'\t\t\t\t\t"Input image is too small (Width: " + std::to_string(inputImage.cols) +\r\n'
b'\t\t\t\t\t", Height: " + std::to_string(inputImage.rows) + ")",\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Populate the input vectors\r\n'
b'\t\t\tconst auto& inputDims = m_trtEngine->getInputDims();\r\n'
b'\t\t\tconst int inputH = inputDims[0].d[1];\r\n'
b'\t\t\tconst int inputW = inputDims[0].d[2];\r\n'
b'\r\n'
b'\t\t\t// Upload the image to GPU memory\r\n'
b'\t\t\tcv::cuda::Stream stream; // Create a custom stream\r\n'
b'\t\t\tcv::cuda::GpuMat img;\r\n'
b'\r\n'
b'\t\t\tif (inputImage.channels() == 1) {\r\n'
b'\t\t\t\t// Convert grayscale to 3-channel BGR before uploading\r\n'
b'\t\t\t\tcv::Mat img3Channel;\r\n'
b'\t\t\t\tcv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);\r\n'
b'\t\t\t\timg.upload(img3Channel, stream);\r\n'
b'\t\t\t}\r\n'
b'\t\t\telse {\r\n'
b'\t\t\t\timg.upload(inputImage, stream);\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Convert BGR to RGB\r\n'
b'\t\t\tcv::cuda::GpuMat imgRGB;\r\n'
b'\t\t\tcv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);\r\n'
b'\t\t\tstream.waitForCompletion();\r\n'
b'\r\n'
b'\t\t\t// These parameters will be used in the post-processing stage\r\n'
b'\t\t\toutMeta.imgHeight = imgRGB.rows;\r\n'
b'\t\t\toutMeta.imgWidth = imgRGB.cols;\r\n'
b'\r\n'
b'\t\t\tif (outMeta.imgHeight <= 0 || outMeta.imgWidth <= 0) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::Preprocess", "Image height or width is zero", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tint maxImageSize = std::max(outMeta.imgHeight, outMeta.imgWidth);\r\n'
b'\t\t\tcv::cuda::GpuMat processed = imgRGB;\r\n'
b'\r\n'
b'\t\t\tif (outMeta.imgHeight > 0 && outMeta.imgWidth > 0) {\r\n'
b'\t\t\t\toutMeta.ratio = 1.f / std::min(inputDims[0].d[2] / static_cast<float>(imgRGB.cols),\r\n'
b'\t\t\t\t\tinputDims[0].d[1] / static_cast<float>(imgRGB.rows));\r\n'
b'\r\n'
b'\t\t\t\tcv::cuda::GpuMat resized = imgRGB;\r\n'
b'\r\n'
b'\t\t\t\t// Resize to the model\'s expected input size while maintaining aspect ratio with padding\r\n'
b'\t\t\t\tif (resized.rows != inputDims[0].d[1] || resized.cols != inputDims[0].d[2]) {\r\n'
b'\t\t\t\t\tresized = Engine<float>::resizeKeepAspectRatioPadRightBottom(imgRGB, inputDims[0].d[1], inputDims[0].d[2]);\r\n'
b'\t\t\t\t}\r\n'
b'\r\n'
b'\t\t\t\t// Convert to format expected by our inference engine\r\n'
b'\t\t\t\tstd::vector<cv::cuda::GpuMat> input{ std::move(resized) };\r\n'
b'\t\t\t\tstd::vector<std::vector<cv::cuda::GpuMat>> inputs{ std::move(input) };\r\n'
b'\t\t\t\treturn inputs;\r\n'
b'\t\t\t}\r\n'
b'\t\t\telse {\r\n'
b'\t\t\t\tthis->_logger.LogFatal("TENSORRTCL::Preprocess",\r\n'
b'\t\t\t\t\t"Image height or width is zero after processing (Width: " + std::to_string(outMeta.imgWidth) +\r\n'
b'\t\t\t\t\t", Height: " + std::to_string(outMeta.imgHeight) + ")",\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\t\t}\r\n'
b'\t\tcatch (const std::exception& e) {\r\n'
b'\t\t\tthis->_logger.LogFatal("TENSORRTCL::Preprocess", e.what(), __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\t}\r\n'
b'\t',
'Step 6: Preprocess outMeta')
print('Step 6: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 7: Postprocess — add meta param, remove lock, replace member refs
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::Postprocess(std::vector<float>& featureVector, const std::string& camera_id) {\r\n',
b'\tstd::vector<std::vector<Object>> TENSORRTCL::DetectObjectsBatch(',
b'TENSORRTCL::Postprocess(std::vector<float>& featureVector, const std::string& camera_id, const ImageMetadata& meta) {\r\n'
b'\t\tstd::vector<Object> outputs;\r\n'
b'\t\ttry {\r\n'
b'\t\t\tauto max_idx = std::max_element(featureVector.begin(), featureVector.end());\r\n'
b'\t\t\tint class_id = static_cast<int>(std::distance(featureVector.begin(), max_idx));\r\n'
b'\t\t\tfloat score = *max_idx;\r\n'
b'\t\t\tint classNameSize = _classes.size();\r\n'
b'\t\t\tObject clsResult;\r\n'
b'\t\t\tclsResult.classId = class_id;\r\n'
b'\t\t\tif (!_classes.empty()) {\r\n'
b'\t\t\t\tif (clsResult.classId < classNameSize) {\r\n'
b'\t\t\t\t\tclsResult.className = _classes[clsResult.classId];\r\n'
b'\t\t\t\t}\r\n'
b'\t\t\t\telse {\r\n'
b'\t\t\t\t\tclsResult.className = _classes[classNameSize - 1]; // Use last valid class name if out of range\r\n'
b'\t\t\t\t}\r\n'
b'\t\t\t}\r\n'
b'\t\t\telse {\r\n'
b'\t\t\t\tclsResult.className = "Unknown"; // Fallback if _classes is empty\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\r\n'
b'\t\t\tclsResult.confidence = score;\r\n'
b'\t\t\tif (meta.imgWidth > 20 && meta.imgHeight > 20) {\r\n'
b'\t\t\t\tclsResult.box = cv::Rect(10, 10, meta.imgWidth - 20, meta.imgHeight - 20);\r\n'
b'\t\t\t}\r\n'
b'\t\t\telse {\r\n'
b'\t\t\t\tclsResult.box = cv::Rect(0, 0, meta.imgWidth, meta.imgHeight);\r\n'
b'\t\t\t}\r\n'
b'\t\t\tclsResult.polygon = ANSUtilityHelper::RectToNormalizedPolygon(clsResult.box, meta.imgWidth, meta.imgHeight);\r\n'
b'\t\t\tclsResult.cameraId = camera_id;\r\n'
b'\t\t\toutputs.push_back(clsResult);\r\n'
b'\t\t\treturn outputs;\r\n'
b'\t\t\t//EnqueueDetection(objects, camera_id);\r\n'
b'\t\t}\r\n'
b'\t\tcatch (std::exception& e) {\r\n'
b'\t\t\tthis->_logger.LogFatal("TENSORRTCL::Postproces", e.what(), __FILE__, __LINE__);\r\n'
b'\t\t\treturn outputs;\r\n'
b'\t\t}\r\n'
b'\r\n'
b'\t}\r\n'
b'\t',
'Step 7: Postprocess meta')
print('Step 7: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 8: DetectObjectsBatch — auto-split + parallel postprocess, no _isFixedBatch
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages,const std::string& camera_id)\r\n',
b'\tstd::vector<std::vector<cv::cuda::GpuMat>> TENSORRTCL::PreprocessBatch(',
b'TENSORRTCL::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id)\r\n'
b'\t{\r\n'
b'\t\t// Validate under brief lock\r\n'
b'\t\t{\r\n'
b'\t\t\tstd::lock_guard<std::recursive_mutex> lock(_mutex);\r\n'
b'\t\t\tif (inputImages.empty()) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::DetectObjectsBatch",\r\n'
b'\t\t\t\t\t"Empty input images vector", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\t\t}\r\n'
b'\r\n'
b'\t\t// Auto-split if batch exceeds engine capacity\r\n'
b'\t\tconst int maxBatch = m_options.maxBatchSize > 0 ? m_options.maxBatchSize : 1;\r\n'
b'\t\tif (static_cast<int>(inputImages.size()) > maxBatch) {\r\n'
b'\t\t\tconst size_t numImages = inputImages.size();\r\n'
b'\t\t\tstd::vector<std::vector<cv::Mat>> chunks;\r\n'
b'\t\t\tfor (size_t start = 0; start < numImages; start += static_cast<size_t>(maxBatch)) {\r\n'
b'\t\t\t\tconst size_t end = std::min(start + static_cast<size_t>(maxBatch), numImages);\r\n'
b'\t\t\t\tchunks.emplace_back(inputImages.begin() + start, inputImages.begin() + end);\r\n'
b'\t\t\t}\r\n'
b'\t\t\tstd::vector<std::future<std::vector<std::vector<Object>>>> futures;\r\n'
b'\t\t\tfutures.reserve(chunks.size());\r\n'
b'\t\t\tfor (size_t i = 0; i < chunks.size(); ++i) {\r\n'
b'\t\t\t\tfutures.push_back(std::async(std::launch::async,\r\n'
b'\t\t\t\t\t[this, c = chunks[i], cid = camera_id]() {\r\n'
b'\t\t\t\t\t\treturn DetectObjectsBatch(c, cid);\r\n'
b'\t\t\t\t\t}));\r\n'
b'\t\t\t}\r\n'
b'\t\t\tstd::vector<std::vector<Object>> allResults;\r\n'
b'\t\t\tallResults.reserve(numImages);\r\n'
b'\t\t\tfor (auto& fut : futures) {\r\n'
b'\t\t\t\tauto chunkResults = fut.get();\r\n'
b'\t\t\t\tfor (auto& r : chunkResults) allResults.push_back(std::move(r));\r\n'
b'\t\t\t}\r\n'
b'\t\t\treturn allResults;\r\n'
b'\t\t}\r\n'
b'\r\n'
b'\t\t_logger.LogInfo("TENSORRTCL::DetectObjectsBatch",\r\n'
b'\t\t\t"Processing batch of " + std::to_string(inputImages.size()) + " images",\r\n'
b'\t\t\t__FILE__, __LINE__);\r\n'
b'\r\n'
b'\t\t// Phase 1: Preprocess under brief lock\r\n'
b'\t\tBatchMetadata metadata;\r\n'
b'\t\tstd::vector<std::vector<cv::cuda::GpuMat>> inputs;\r\n'
b'\t\t{\r\n'
b'\t\t\tstd::lock_guard<std::recursive_mutex> lock(_mutex);\r\n'
b'\t\t\tinputs = PreprocessBatch(inputImages, metadata);\r\n'
b'\t\t}\r\n'
b'\t\tif (inputs.empty() || inputs[0].empty()) {\r\n'
b'\t\t\t_logger.LogFatal("TENSORRTCL::DetectObjectsBatch",\r\n'
b'\t\t\t\t"Preprocessing failed", __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\r\n'
b'\t\t// Phase 2: Inference \xe2\x80\x94 mutex released; pool dispatches to idle GPU slot\r\n'
b'\t\tstd::vector<std::vector<std::vector<float>>> featureVectors;\r\n'
b'\t\tbool succ = m_trtEngine->runInference(inputs, featureVectors);\r\n'
b'\t\tif (!succ) {\r\n'
b'\t\t\t_logger.LogFatal("TENSORRTCL::DetectObjectsBatch",\r\n'
b'\t\t\t\t"Error running batch inference", __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\r\n'
b'\t\t// Phase 3: Parallel postprocessing\r\n'
b'\t\tconst size_t numBatch = featureVectors.size();\r\n'
b'\t\tstd::vector<std::vector<Object>> batchDetections(numBatch);\r\n'
b'\t\tstd::vector<std::future<std::vector<Object>>> postFutures;\r\n'
b'\t\tpostFutures.reserve(numBatch);\r\n'
b'\r\n'
b'\t\tfor (size_t batchIdx = 0; batchIdx < numBatch; ++batchIdx) {\r\n'
b'\t\t\tconst auto& batchOutput = featureVectors[batchIdx];\r\n'
b'\t\t\tstd::vector<float> fv = batchOutput.empty() ? std::vector<float>{} : batchOutput[0];\r\n'
b'\t\t\tpostFutures.push_back(std::async(std::launch::async,\r\n'
b'\t\t\t\t[this, fv = std::move(fv), cid = camera_id, idx = batchIdx, &metadata]() mutable {\r\n'
b'\t\t\t\t\treturn PostprocessBatch(fv, cid, idx, metadata);\r\n'
b'\t\t\t\t}));\r\n'
b'\t\t}\r\n'
b'\t\tfor (size_t i = 0; i < numBatch; ++i)\r\n'
b'\t\t\tbatchDetections[i] = postFutures[i].get();\r\n'
b'\r\n'
b'\t\t_logger.LogInfo("TENSORRTCL::DetectObjectsBatch",\r\n'
b'\t\t\t"Batch processing complete. Images: " + std::to_string(numBatch),\r\n'
b'\t\t\t__FILE__, __LINE__);\r\n'
b'\t\treturn batchDetections;\r\n'
b'\t}\r\n'
b'\t',
'Step 8: DetectObjectsBatch rewrite')
print('Step 8: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 9: PreprocessBatch — add outMetadata param, remove lock, rename members
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::PreprocessBatch(const std::vector<cv::Mat>& inputImages)\r\n',
b'\tstd::vector<Object> TENSORRTCL::PostprocessBatch(',
b'TENSORRTCL::PreprocessBatch(const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata)\r\n'
b'\t{\r\n'
b'\t\ttry {\r\n'
b'\t\t\t// Validate license\r\n'
b'\t\t\tif (!_licenseValid) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\t\t"Invalid license", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Validate input\r\n'
b'\t\t\tif (inputImages.empty()) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\t\t"Input images vector is empty", __FILE__, __LINE__);\r\n'
b'\t\t\t\treturn {};\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tsize_t batchSize = inputImages.size();\r\n'
b'\r\n'
b'\t\t\t// Get model input dimensions\r\n'
b'\t\t\tconst auto& inputDims = m_trtEngine->getInputDims();\r\n'
b'\t\t\tconst int inputH = inputDims[0].d[1];\r\n'
b'\t\t\tconst int inputW = inputDims[0].d[2];\r\n'
b'\r\n'
b'\t\t\t_logger.LogInfo("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\t"Preprocessing " + std::to_string(batchSize) + " images to " +\r\n'
b'\t\t\t\tstd::to_string(inputW) + "x" + std::to_string(inputH),\r\n'
b'\t\t\t\t__FILE__, __LINE__);\r\n'
b'\r\n'
b'\t\t\t// Create CUDA stream for async operations\r\n'
b'\t\t\tcv::cuda::Stream stream;\r\n'
b'\r\n'
b'\t\t\t// Store ALL images in a SINGLE batch vector\r\n'
b'\t\t\tstd::vector<cv::cuda::GpuMat> batchedImages;\r\n'
b'\t\t\tbatchedImages.reserve(batchSize);\r\n'
b'\r\n'
b'\t\t\t// Store image dimensions for postprocessing\r\n'
b'\t\t\toutMetadata.imgHeights.clear();\r\n'
b'\t\t\toutMetadata.imgWidths.clear();\r\n'
b'\t\t\toutMetadata.ratios.clear();\r\n'
b'\t\t\toutMetadata.imgHeights.reserve(batchSize);\r\n'
b'\t\t\toutMetadata.imgWidths.reserve(batchSize);\r\n'
b'\t\t\toutMetadata.ratios.reserve(batchSize);\r\n'
b'\r\n'
b'\t\t\t// Process each image\r\n'
b'\t\t\tfor (size_t i = 0; i < batchSize; ++i) {\r\n'
b'\t\t\t\tconst cv::Mat& inputImage = inputImages[i];\r\n'
b'\r\n'
b'\t\t\t\t// Validate individual image\r\n'
b'\t\t\t\tif (inputImage.empty()) {\r\n'
b'\t\t\t\t\t_logger.LogFatal("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\t\t\t"Input image at index " + std::to_string(i) + " is empty",\r\n'
b'\t\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\t\treturn {};\r\n'
b'\t\t\t\t}\r\n'
b'\r\n'
b'\t\t\t\tif (inputImage.cols < 5 || inputImage.rows < 5) {\r\n'
b'\t\t\t\t\t_logger.LogFatal("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\t\t\t"Image at index " + std::to_string(i) +\r\n'
b'\t\t\t\t\t\t" is too small (Width: " + std::to_string(inputImage.cols) +\r\n'
b'\t\t\t\t\t\t", Height: " + std::to_string(inputImage.rows) + ")",\r\n'
b'\t\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\t\treturn {};\r\n'
b'\t\t\t\t}\r\n'
b'\r\n'
b'\t\t\t\t// Upload to GPU\r\n'
b'\t\t\t\tcv::cuda::GpuMat img;\r\n'
b'\t\t\t\tif (inputImage.channels() == 1) {\r\n'
b'\t\t\t\t\t// Convert grayscale to BGR\r\n'
b'\t\t\t\t\tcv::Mat img3Channel;\r\n'
b'\t\t\t\t\tcv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);\r\n'
b'\t\t\t\t\timg.upload(img3Channel, stream);\r\n'
b'\t\t\t\t}\r\n'
b'\t\t\t\telse {\r\n'
b'\t\t\t\t\timg.upload(inputImage, stream);\r\n'
b'\t\t\t\t}\r\n'
b'\r\n'
b'\t\t\t\t// Convert BGR to RGB\r\n'
b'\t\t\t\tcv::cuda::GpuMat imgRGB;\r\n'
b'\t\t\t\tcv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);\r\n'
b'\r\n'
b'\t\t\t\t// Store original dimensions\r\n'
b'\t\t\t\tint imgHeight = imgRGB.rows;\r\n'
b'\t\t\t\tint imgWidth = imgRGB.cols;\r\n'
b'\r\n'
b'\t\t\t\tif (imgHeight <= 0 || imgWidth <= 0) {\r\n'
b'\t\t\t\t\t_logger.LogFatal("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\t\t\t"Image at index " + std::to_string(i) + " has zero height or width",\r\n'
b'\t\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\t\treturn {};\r\n'
b'\t\t\t\t}\r\n'
b'\r\n'
b'\t\t\t\toutMetadata.imgHeights.push_back(imgHeight);\r\n'
b'\t\t\t\toutMetadata.imgWidths.push_back(imgWidth);\r\n'
b'\r\n'
b'\t\t\t\t// Calculate resize ratio\r\n'
b'\t\t\t\tfloat ratio = 1.f / std::min(\r\n'
b'\t\t\t\t\tinputDims[0].d[2] / static_cast<float>(imgRGB.cols),\r\n'
b'\t\t\t\t\tinputDims[0].d[1] / static_cast<float>(imgRGB.rows)\r\n'
b'\t\t\t\t);\r\n'
b'\t\t\t\toutMetadata.ratios.push_back(ratio);\r\n'
b'\r\n'
b'\t\t\t\t// Resize maintaining aspect ratio with padding\r\n'
b'\t\t\t\tcv::cuda::GpuMat resized;\r\n'
b'\t\t\t\tif (imgRGB.rows != inputDims[0].d[1] || imgRGB.cols != inputDims[0].d[2]) {\r\n'
b'\t\t\t\t\tresized = Engine<float>::resizeKeepAspectRatioPadRightBottom(\r\n'
b'\t\t\t\t\t\timgRGB, inputDims[0].d[1], inputDims[0].d[2]\r\n'
b'\t\t\t\t\t);\r\n'
b'\t\t\t\t}\r\n'
b'\t\t\t\telse {\r\n'
b'\t\t\t\t\tresized = imgRGB;\r\n'
b'\t\t\t\t}\r\n'
b'\r\n'
b'\t\t\t\t// Add to batch\r\n'
b'\t\t\t\tbatchedImages.push_back(std::move(resized));\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Wait for all GPU operations to complete\r\n'
b'\t\t\tstream.waitForCompletion();\r\n'
b'\r\n'
b'\t\t\t// Return as single batched input\r\n'
b'\t\t\tstd::vector<std::vector<cv::cuda::GpuMat>> result;\r\n'
b'\t\t\tresult.push_back(std::move(batchedImages));\r\n'
b'\r\n'
b'\t\t\treturn result;\r\n'
b'\t\t}\r\n'
b'\t\tcatch (const std::exception& e) {\r\n'
b'\t\t\t_logger.LogFatal("TENSORRTCL::PreprocessBatch",\r\n'
b'\t\t\t\te.what(), __FILE__, __LINE__);\r\n'
b'\t\t\treturn {};\r\n'
b'\t\t}\r\n'
b'\t}\r\n'
b'\t',
'Step 9: PreprocessBatch outMetadata')
print('Step 9: done')
# ──────────────────────────────────────────────────────────────────────────────
# Step 10: PostprocessBatch — add metadata param, remove lock, rename members
# ──────────────────────────────────────────────────────────────────────────────
data = replace_slice(data,
b'TENSORRTCL::PostprocessBatch(std::vector<float>& featureVector,const std::string& camera_id,size_t batchIdx)\r\n',
b'\t\r\n}',
b'TENSORRTCL::PostprocessBatch(std::vector<float>& featureVector, const std::string& camera_id, size_t batchIdx, const BatchMetadata& metadata)\r\n'
b'\t{\r\n'
b'\t\tstd::vector<Object> outputs;\r\n'
b'\r\n'
b'\t\ttry {\r\n'
b'\t\t\t// Validate batch index\r\n'
b'\t\t\tif (batchIdx >= metadata.imgHeights.size() ||\r\n'
b'\t\t\t\tbatchIdx >= metadata.imgWidths.size()) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::PostprocessBatch",\r\n'
b'\t\t\t\t\t"Batch index " + std::to_string(batchIdx) +\r\n'
b'\t\t\t\t\t" out of range (stored " + std::to_string(metadata.imgHeights.size()) + " images)",\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn outputs;\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Validate feature vector\r\n'
b'\t\t\tif (featureVector.empty()) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::PostprocessBatch",\r\n'
b'\t\t\t\t\t"Feature vector is empty for batch index " + std::to_string(batchIdx),\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn outputs;\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Get image dimensions for this batch index\r\n'
b'\t\t\tint imgHeight = metadata.imgHeights[batchIdx];\r\n'
b'\t\t\tint imgWidth = metadata.imgWidths[batchIdx];\r\n'
b'\r\n'
b'\t\t\t// Find max element (classification result)\r\n'
b'\t\t\tauto max_idx = std::max_element(featureVector.begin(), featureVector.end());\r\n'
b'\t\t\tif (max_idx == featureVector.end()) {\r\n'
b'\t\t\t\t_logger.LogFatal("TENSORRTCL::PostprocessBatch",\r\n'
b'\t\t\t\t\t"Failed to find max element in feature vector for batch index " +\r\n'
b'\t\t\t\t\tstd::to_string(batchIdx),\r\n'
b'\t\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\t\treturn outputs;\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tint class_id = static_cast<int>(std::distance(featureVector.begin(), max_idx));\r\n'
b'\t\t\tfloat score = *max_idx;\r\n'
b'\r\n'
b'\t\t\t// Create object result\r\n'
b'\t\t\tObject clsResult;\r\n'
b'\t\t\tclsResult.classId = class_id;\r\n'
b'\r\n'
b'\t\t\t// Get class name\r\n'
b'\t\t\tint classNameSize = static_cast<int>(_classes.size());\r\n'
b'\t\t\tif (!_classes.empty()) {\r\n'
b'\t\t\t\tif (class_id >= 0 && class_id < classNameSize) {\r\n'
b'\t\t\t\t\tclsResult.className = _classes[class_id];\r\n'
b'\t\t\t\t}\r\n'
b'\t\t\t\telse {\r\n'
b'\t\t\t\t\tclsResult.className = _classes[classNameSize - 1];\r\n'
b'\r\n'
b'\t\t\t\t}\r\n'
b'\t\t\t}\r\n'
b'\t\t\telse {\r\n'
b'\t\t\t\tclsResult.className = "Unknown";\r\n'
b'\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\tclsResult.confidence = score;\r\n'
b'\r\n'
b'\t\t\t// Create bounding box with margins\r\n'
b'\t\t\tif (imgWidth > 20 && imgHeight > 20) {\r\n'
b'\t\t\t\tclsResult.box = cv::Rect(10, 10, imgWidth - 20, imgHeight - 20);\r\n'
b'\t\t\t}\r\n'
b'\t\t\telse {\r\n'
b'\t\t\t\tclsResult.box = cv::Rect(0, 0, imgWidth, imgHeight);\r\n'
b'\t\t\t}\r\n'
b'\r\n'
b'\t\t\t// Convert to normalized polygon\r\n'
b'\t\t\tclsResult.polygon = ANSUtilityHelper::RectToNormalizedPolygon(\r\n'
b'\t\t\t\tclsResult.box, imgWidth, imgHeight\r\n'
b'\t\t\t);\r\n'
b'\r\n'
b'\t\t\tclsResult.cameraId = camera_id;\r\n'
b'\r\n'
b'\t\t\toutputs.push_back(std::move(clsResult));\r\n'
b'\r\n'
b'\t\t\treturn outputs;\r\n'
b'\t\t}\r\n'
b'\t\tcatch (const std::exception& e) {\r\n'
b'\t\t\t_logger.LogFatal("TENSORRTCL::PostprocessBatch",\r\n'
b'\t\t\t\t"Error for batch index " + std::to_string(batchIdx) + ": " + e.what(),\r\n'
b'\t\t\t\t__FILE__, __LINE__);\r\n'
b'\t\t\treturn outputs;\r\n'
b'\t\t}\r\n'
b'\t}\r\n',
'Step 10: PostprocessBatch metadata')
print('Step 10: done')
# ──────────────────────────────────────────────────────────────────────────────
# Verification: check no old member names remain
# ──────────────────────────────────────────────────────────────────────────────
stale = [b'm_ratio', b'm_imgWidth', b'm_imgHeight',
b'm_batchImgHeights', b'm_batchImgWidths', b'm_batchRatios',
b'_isFixedBatch']
for s in stale:
count = data.count(s)
if count > 0:
errors.append(f'Stale member {s.decode()} still appears {count}x')
# ──────────────────────────────────────────────────────────────────────────────
# Result
# ──────────────────────────────────────────────────────────────────────────────
if errors:
print('\nERRORS:')
for e in errors:
print(' ', e)
print('\nFile NOT written.')
sys.exit(1)
else:
with open(CPP, 'wb') as f:
f.write(data)
print(f'\nAll OK. File written ({len(data)} bytes).')
+492
View File
@@ -0,0 +1,492 @@
import sys
CPP = r'C:\Projects\ANLS\ANSLIB\ANSODEngine\ANSTENSORRTPOSE.cpp'
with open(CPP, 'rb') as f:
data = f.read()
CRLF = b'\r\n' in data
print(f'Line ending: {"CRLF" if CRLF else "LF"}')
def L(s):
if CRLF:
return s.replace(b'\n', b'\r\n')
return s
errors = []
def replace_once(data, old, new, label):
count = data.count(old)
if count != 1:
errors.append(f'ERROR [{label}]: expected 1, found {count}')
print(errors[-1])
return data
print(f'OK [{label}]')
return data.replace(old, new, 1)
def replace_all(data, old, new, expected, label):
count = data.count(old)
if count != expected:
errors.append(f'ERROR [{label}]: expected {expected}, found {count}')
print(errors[-1])
return data
print(f'OK [{label}] ({count} replacements)')
return data.replace(old, new)
def replace_slice(data, start_anchor, end_anchor, new_content, label):
"""Find start_anchor, then end_anchor after it, replace [start..end+len] with new_content."""
s = data.find(start_anchor)
if s == -1:
errors.append(f'ERROR [{label}]: start_anchor not found: {start_anchor[:50]!r}')
print(errors[-1])
return data
e = data.find(end_anchor, s + len(start_anchor))
if e == -1:
errors.append(f'ERROR [{label}]: end_anchor not found: {end_anchor[:50]!r}')
print(errors[-1])
return data
e += len(end_anchor)
old_chunk = data[s:e]
if data.count(old_chunk) != 1:
errors.append(f'ERROR [{label}]: extracted chunk not unique (count={data.count(old_chunk)})')
print(errors[-1])
return data
print(f'OK [{label}]')
return data[:s] + new_content + data[e:]
orig_size = len(data)
# 1. Add #include <future>
data = replace_once(data,
L(b'#include <opencv2/cudaimgproc.hpp>\n'),
L(b'#include <opencv2/cudaimgproc.hpp>\n#include <future>\n'),
'add future')
# 2. Three buildLoadNetwork calls -> add m_maxSlotsPerGpu
data = replace_all(data,
b'buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE)',
b'buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu)',
3, 'buildLoadNetwork 3x')
# 3. RunInference(camera_id) - release mutex before DetectObjects
# Old body: single lock scope containing all validation then try/catch
# New body: brief lock scope for validation, then try { return DetectObjects; } catch { ... }
old_ri = L(
b' std::vector<Object> ANSTENSORRTPOSE::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b'\n'
b' // Validation checks\n'
b' if (!_modelLoadValid) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInference",\n'
b' "Cannot load the TensorRT model. Please check if it exists",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (!_licenseValid) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInference",\n'
b' "Runtime license is not valid or expired. Please contact ANSCENTER",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (!_isInitialized) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInference",\n'
b' "Model is not initialized",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) {\n'
b' return {};\n'
b' }\n'
b'\n'
b' try {\n'
b' return DetectObjects(inputImgBGR, camera_id);\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::RunInference", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n')
new_ri = L(
b' std::vector<Object> ANSTENSORRTPOSE::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)\n'
b' {\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b'\n'
b' // Validation checks\n'
b' if (!_modelLoadValid) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInference",\n'
b' "Cannot load the TensorRT model. Please check if it exists",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (!_licenseValid) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInference",\n'
b' "Runtime license is not valid or expired. Please contact ANSCENTER",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (!_isInitialized) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInference",\n'
b' "Model is not initialized",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) {\n'
b' return {};\n'
b' }\n'
b' }\n'
b' try {\n'
b' return DetectObjects(inputImgBGR, camera_id);\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::RunInference", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n')
data = replace_once(data, old_ri, new_ri, 'RunInference refactor')
# 4. DetectObjects - three-phase mutex
# Old body: single lock, try { Preprocess -> runInference -> if(succ) PostProcessPose }
# New body: Phase1 lock+Preprocess, Phase2 inference, Phase3 lock+PostProcessPose
old_det = L(
b' // private\n'
b' std::vector<Object> ANSTENSORRTPOSE::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' try {\n'
b' // Preprocess the input image\n'
b' const auto input = Preprocess(inputImage);\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(input, featureVectors);\n'
b' if (succ) {\n'
b' // Check if our model does only object detection or also supports segmentation\n'
b' std::vector<Object> ret;\n'
b' const auto& numOutputs = m_trtEngine->getOutputDims().size();\n'
b' std::vector<float> featureVector;\n'
b' Engine<float>::transformOutput(featureVectors, featureVector);\n'
b' const auto& outputDims = m_trtEngine->getOutputDims();\n'
b' int numChannels = outputDims[outputDims.size() - 1].d[1];\n'
b' ret = PostProcessPose(featureVector, camera_id);\n'
b' return ret;\n'
b' }\n'
b' else {\n'
b' this->_logger.LogError("ANSTENSORRTPOSE::DetectObjects", "Error running inference", __FILE__, __LINE__);\n'
b' std::vector<Object> ret;\n'
b' ret.clear();\n'
b' return ret;\n'
b' }\n'
b' }\n'
b' catch (std::exception& e) {\n'
b' this->_logger.LogFatal("ANSTENSORRTPOSE::DetectObjects", e.what(), __FILE__, __LINE__);\n'
b' std::vector<Object> ret;\n'
b' ret.clear();\n'
b' return ret;\n'
b' }\n'
b' }\n')
new_det = L(
b' // private\n'
b' std::vector<Object> ANSTENSORRTPOSE::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {\n'
b' // Phase 1: Preprocess under lock\n'
b' ImageMetadata meta;\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> input;\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' input = Preprocess(inputImage, meta);\n'
b' }\n'
b' if (input.empty()) return {};\n'
b'\n'
b' // Phase 2: Inference -- mutex released; pool dispatches to idle GPU slot\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(input, featureVectors);\n'
b' if (!succ) {\n'
b' this->_logger.LogError("ANSTENSORRTPOSE::DetectObjects", "Error running inference", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Phase 3: Postprocess under lock\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' std::vector<float> featureVector;\n'
b' Engine<float>::transformOutput(featureVectors, featureVector);\n'
b' return PostProcessPose(featureVector, camera_id, meta);\n'
b' }\n')
data = replace_once(data, old_det, new_det, 'DetectObjects refactor')
# 5a. Preprocess - remove inner lock, add outMeta param
old_pre_sig = L(
b' std::vector<std::vector<cv::cuda::GpuMat>> ANSTENSORRTPOSE::Preprocess(const cv::Mat& inputImage) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n')
new_pre_sig = L(
b' std::vector<std::vector<cv::cuda::GpuMat>> ANSTENSORRTPOSE::Preprocess(const cv::Mat& inputImage, ImageMetadata& outMeta) {\n')
data = replace_once(data, old_pre_sig, new_pre_sig, 'Preprocess sig + remove lock')
# 5b. Preprocess member writes -> outMeta
# Actual file has blank line + "// Set image size parameters" comment before the member writes
old_pw = L(
b' stream.waitForCompletion();\n'
b'\n'
b' // Set image size parameters\n'
b' m_imgHeight = imgRGB.rows;\n'
b' m_imgWidth = imgRGB.cols;\n'
b' if (m_imgHeight > 0 && m_imgWidth > 0) {\n'
b' m_ratio = 1.f / std::min(inputDims[0].d[2] / static_cast<float>(imgRGB.cols),\n'
b' inputDims[0].d[1] / static_cast<float>(imgRGB.rows));\n')
new_pw = L(
b' stream.waitForCompletion();\n'
b'\n'
b' // Set image size parameters\n'
b' outMeta.imgHeight = imgRGB.rows;\n'
b' outMeta.imgWidth = imgRGB.cols;\n'
b' if (outMeta.imgHeight > 0 && outMeta.imgWidth > 0) {\n'
b' outMeta.ratio = 1.f / std::min(inputDims[0].d[2] / static_cast<float>(imgRGB.cols),\n'
b' inputDims[0].d[1] / static_cast<float>(imgRGB.rows));\n')
data = replace_once(data, old_pw, new_pw, 'Preprocess member writes')
# 5c. Preprocess error log
old_pe = L(
b' this->_logger.LogFatal("TENSORRTCL::Preprocess",\n'
b' "Image height or width is zero after processing (Width: " + std::to_string(m_imgWidth) +\n'
b' ", Height: " + std::to_string(m_imgHeight) + ")",\n')
new_pe = L(
b' this->_logger.LogFatal("TENSORRTCL::Preprocess",\n'
b' "Image height or width is zero after processing (Width: " + std::to_string(outMeta.imgWidth) +\n'
b' ", Height: " + std::to_string(outMeta.imgHeight) + ")",\n')
data = replace_once(data, old_pe, new_pe, 'Preprocess error log')
# 6a. PostProcessPose - remove lock, add const ImageMetadata& meta param
old_ppp_sig = L(
b' std::vector<Object> ANSTENSORRTPOSE::PostProcessPose(std::vector<float>& featureVector, const std::string& camera_id) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n')
new_ppp_sig = L(
b' std::vector<Object> ANSTENSORRTPOSE::PostProcessPose(std::vector<float>& featureVector, const std::string& camera_id, const ImageMetadata& meta) {\n')
data = replace_once(data, old_ppp_sig, new_ppp_sig, 'PostProcessPose sig + remove lock')
# 6b/c/d. Replace m_ratio/m_imgWidth/m_imgHeight -> meta.* in PostProcessPose
# After steps 5b+5c fix Preprocess (removes 1/3/3), remaining counts: 6/4/4 (all in PostProcessPose)
data = replace_all(data, b'm_ratio', b'meta.ratio', 6, 'meta.ratio (6x in PostProcessPose)')
data = replace_all(data, b'm_imgWidth', b'meta.imgWidth', 4, 'meta.imgWidth (4x in PostProcessPose)')
data = replace_all(data, b'm_imgHeight', b'meta.imgHeight', 4, 'meta.imgHeight (4x in PostProcessPose)')
# 7. Rewrite DetectObjectsBatch + PreprocessBatch + PostProcessPoseBatch beginning
# Use anchor-based slice replacement to handle the exact bytes without hardcoding the full block
new_batch_all = L(
b' std::vector<std::vector<Object>> ANSTENSORRTPOSE::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id) {\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' if (inputImages.empty()) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::DetectObjectsBatch", "Empty input images vector", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b'\n'
b' // Auto-split if batch exceeds engine capacity\n'
b' const int maxBatch = m_options.maxBatchSize > 0 ? m_options.maxBatchSize : 1;\n'
b' if (static_cast<int>(inputImages.size()) > maxBatch) {\n'
b' const size_t numImages = inputImages.size();\n'
b' std::vector<std::vector<cv::Mat>> chunks;\n'
b' for (size_t start = 0; start < numImages; start += static_cast<size_t>(maxBatch)) {\n'
b' const size_t end = std::min(start + static_cast<size_t>(maxBatch), numImages);\n'
b' chunks.emplace_back(inputImages.begin() + start, inputImages.begin() + end);\n'
b' }\n'
b' std::vector<std::future<std::vector<std::vector<Object>>>> futures;\n'
b' futures.reserve(chunks.size());\n'
b' for (size_t i = 0; i < chunks.size(); ++i) {\n'
b' futures.push_back(std::async(std::launch::async,\n'
b' [this, c = chunks[i], cid = camera_id]() {\n'
b' return DetectObjectsBatch(c, cid);\n'
b' }));\n'
b' }\n'
b' std::vector<std::vector<Object>> allResults;\n'
b' allResults.reserve(numImages);\n'
b' for (auto& fut : futures) {\n'
b' auto chunkResults = fut.get();\n'
b' for (auto& r : chunkResults) allResults.push_back(std::move(r));\n'
b' }\n'
b' return allResults;\n'
b' }\n'
b'\n'
b' _logger.LogInfo("ANSTENSORRTPOSE::DetectObjectsBatch",\n'
b' "Processing batch of " + std::to_string(inputImages.size()) + " images", __FILE__, __LINE__);\n'
b'\n'
b' // Phase 1: Preprocess under brief lock\n'
b' BatchMetadata metadata;\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> inputs;\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' inputs = PreprocessBatch(inputImages, metadata);\n'
b' }\n'
b' if (inputs.empty() || inputs[0].empty()) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::DetectObjectsBatch", "Preprocessing failed", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Phase 2: Inference -- mutex released; pool dispatches to idle GPU slot\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(inputs, featureVectors);\n'
b' if (!succ) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::DetectObjectsBatch", "Error running inference", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Phase 3: Parallel postprocessing -- each image is independent\n'
b' const size_t numBatch = featureVectors.size();\n'
b' std::vector<std::vector<Object>> batchDetections(numBatch);\n'
b' std::vector<std::future<std::vector<Object>>> postFutures;\n'
b' postFutures.reserve(numBatch);\n'
b' for (size_t batchIdx = 0; batchIdx < numBatch; ++batchIdx) {\n'
b' const auto& batchOutput = featureVectors[batchIdx];\n'
b' std::vector<float> fv =\n'
b' batchOutput.empty() ? std::vector<float>{} : batchOutput[0];\n'
b' postFutures.push_back(std::async(std::launch::async,\n'
b' [this, fv = std::move(fv), cid = camera_id,\n'
b' idx = batchIdx, &metadata]() mutable {\n'
b' return PostProcessPoseBatch(fv, cid, idx, metadata);\n'
b' }));\n'
b' }\n'
b' for (size_t i = 0; i < numBatch; ++i)\n'
b' batchDetections[i] = postFutures[i].get();\n'
b'\n'
b' _logger.LogInfo("ANSTENSORRTPOSE::DetectObjectsBatch",\n'
b' "Batch processing complete. Images: " + std::to_string(numBatch), __FILE__, __LINE__);\n'
b' return batchDetections;\n'
b' }\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> ANSTENSORRTPOSE::PreprocessBatch(const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata) {\n'
b' try {\n'
b' if (!_licenseValid) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::PreprocessBatch", "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' const auto& inputDims = m_trtEngine->getInputDims();\n'
b' const int inputH = inputDims[0].d[1];\n'
b' const int inputW = inputDims[0].d[2];\n'
b' outMetadata.imgHeights.resize(inputImages.size());\n'
b' outMetadata.imgWidths.resize(inputImages.size());\n'
b' outMetadata.ratios.resize(inputImages.size());\n'
b' std::vector<cv::cuda::GpuMat> batchProcessed;\n'
b' batchProcessed.reserve(inputImages.size());\n'
b' cv::cuda::Stream stream;\n'
b' for (size_t i = 0; i < inputImages.size(); ++i) {\n'
b' const auto& inputImage = inputImages[i];\n'
b' if (inputImage.empty()) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::PreprocessBatch",\n'
b' "Empty input image at index " + std::to_string(i), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' cv::cuda::GpuMat img;\n'
b' if (inputImage.channels() == 1) {\n'
b' cv::Mat img3Channel;\n'
b' cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);\n'
b' img.upload(img3Channel, stream);\n'
b' }\n'
b' else {\n'
b' img.upload(inputImage, stream);\n'
b' }\n'
b' cv::cuda::GpuMat imgRGB;\n'
b' cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);\n'
b' outMetadata.imgHeights[i] = imgRGB.rows;\n'
b' outMetadata.imgWidths[i] = imgRGB.cols;\n'
b' if (outMetadata.imgHeights[i] <= 0 || outMetadata.imgWidths[i] <= 0) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::PreprocessBatch",\n'
b' "Image " + std::to_string(i) + " has invalid dimensions (Width: " +\n'
b' std::to_string(outMetadata.imgWidths[i]) + ", Height: " +\n'
b' std::to_string(outMetadata.imgHeights[i]) + ")", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' outMetadata.ratios[i] = 1.f / std::min(inputW / static_cast<float>(imgRGB.cols),\n'
b' inputH / static_cast<float>(imgRGB.rows));\n'
b' cv::cuda::GpuMat resized = imgRGB;\n'
b' if (resized.rows != inputH || resized.cols != inputW) {\n'
b' resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(imgRGB, inputH, inputW);\n'
b' }\n'
b' batchProcessed.push_back(std::move(resized));\n'
b' }\n'
b' stream.waitForCompletion();\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> inputs;\n'
b' inputs.push_back(std::move(batchProcessed));\n'
b' return inputs;\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::PreprocessBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b' std::vector<Object> ANSTENSORRTPOSE::PostProcessPoseBatch(std::vector<float>& featureVector, const std::string& camera_id, size_t batchIdx, const BatchMetadata& metadata) {\n'
b' try {\n'
b' const auto& outputDims = m_trtEngine->getOutputDims();\n'
b' auto numChannels = outputDims[0].d[1];\n'
b' auto numAnchors = outputDims[0].d[2];\n'
b' float ratio = metadata.ratios[batchIdx];\n'
b' float imgWidth = static_cast<float>(metadata.imgWidths[batchIdx]);\n'
b' float imgHeight = static_cast<float>(metadata.imgHeights[batchIdx]);\n')
data = replace_slice(data,
start_anchor=L(b' std::vector<Object> ANSTENSORRTPOSE::DetectObjectsBatch('),
end_anchor=L(b' float imgHeight = static_cast<float>(m_batchImgHeights[batchIdx]);\n'),
new_content=new_batch_all,
label='DetectObjectsBatch+PreprocessBatch+PostProcessPoseBatch rewrite')
# 8. Add RunInferencesBatch before namespace close
old_ns_close = L(
b' catch (std::exception& e) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::PostProcessPoseBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b'}\n')
new_ns_close = L(
b' catch (std::exception& e) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::PostProcessPoseBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b' std::vector<std::vector<Object>> ANSTENSORRTPOSE::RunInferencesBatch(\n'
b' const std::vector<cv::Mat>& inputs, const std::string& camera_id)\n'
b' {\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' if (!_modelLoadValid) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInferencesBatch",\n'
b' "Model not loaded", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (!_licenseValid) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInferencesBatch",\n'
b' "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (!_isInitialized) {\n'
b' _logger.LogError("ANSTENSORRTPOSE::RunInferencesBatch",\n'
b' "Engine not initialized", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (inputs.empty()) return {};\n'
b' }\n'
b' try {\n'
b' return DetectObjectsBatch(inputs, camera_id);\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("ANSTENSORRTPOSE::RunInferencesBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b'}\n')
data = replace_once(data, old_ns_close, new_ns_close, 'Add RunInferencesBatch + namespace close')
# Summary
print(f'\nOriginal size: {orig_size} bytes')
print(f'New size: {len(data)} bytes')
if errors:
print(f'\n{len(errors)} ERROR(S) -- file NOT written')
sys.exit(1)
else:
with open(CPP, 'wb') as f:
f.write(data)
print('\nOK -- file written')
+677
View File
@@ -0,0 +1,677 @@
import sys
CPP = r'C:\Projects\ANLS\ANSLIB\ANSODEngine\ANSTENSORRTSEG.cpp'
with open(CPP, 'rb') as f:
data = f.read()
CRLF = b'\r\n' in data
print(f'Line ending: {"CRLF" if CRLF else "LF"}')
def L(s):
if CRLF:
return s.replace(b'\n', b'\r\n')
return s
errors = []
def replace_once(data, old, new, label):
count = data.count(old)
if count != 1:
errors.append(f'ERROR [{label}]: expected 1, found {count}')
print(errors[-1])
return data
print(f'OK [{label}]')
return data.replace(old, new, 1)
def replace_all(data, old, new, expected, label):
count = data.count(old)
if count != expected:
errors.append(f'ERROR [{label}]: expected {expected}, found {count}')
print(errors[-1])
return data
print(f'OK [{label}] ({count} replacements)')
return data.replace(old, new)
orig_size = len(data)
# 1. Add #include <future>
data = replace_once(data,
L(b'#include <opencv2/cudaimgproc.hpp>\n'),
L(b'#include <opencv2/cudaimgproc.hpp>\n#include <future>\n'),
'add future')
# 2. Three buildLoadNetwork calls
data = replace_all(data,
b'buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE)',
b'buildLoadNetwork(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu)',
3, 'buildLoadNetwork 3x')
# 3. RunInference(camera_id) - release mutex before DetectObjects
old_ri = L(
b' std::vector<Object> TENSORRTSEG::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b'\n'
b' if (!_modelLoadValid) {\n'
b' _logger.LogError("TENSORRTSEG::RunInference",\n'
b' "Cannot load TensorRT model", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (!_licenseValid) {\n'
b' _logger.LogError("TENSORRTSEG::RunInference",\n'
b' "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (!_isInitialized) {\n'
b' _logger.LogError("TENSORRTSEG::RunInference",\n'
b' "Model not initialized", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) {\n'
b' return {};\n'
b' }\n'
b'\n'
b' try {\n'
b' return DetectObjects(inputImgBGR, camera_id);\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::RunInference", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n')
new_ri = L(
b' std::vector<Object> TENSORRTSEG::RunInference(const cv::Mat& inputImgBGR,const std::string& camera_id)\n'
b' {\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' if (!_modelLoadValid) {\n'
b' _logger.LogError("TENSORRTSEG::RunInference",\n'
b' "Cannot load TensorRT model", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (!_licenseValid) {\n'
b' _logger.LogError("TENSORRTSEG::RunInference",\n'
b' "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (!_isInitialized) {\n'
b' _logger.LogError("TENSORRTSEG::RunInference",\n'
b' "Model not initialized", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) {\n'
b' return {};\n'
b' }\n'
b' }\n'
b' try {\n'
b' return DetectObjects(inputImgBGR, camera_id);\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::RunInference", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n')
data = replace_once(data, old_ri, new_ri, 'RunInference refactor')
# 4. DetectObjects - two-phase mutex
old_det = L(
b' std::vector<Object> TENSORRTSEG::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' try {\n'
b' // Preprocess the input image\n'
b' const auto input = Preprocess(inputImage);\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(input, featureVectors);\n'
b' if (succ) {\n'
b' // Check if our model does only object detection or also supports segmentation\n'
b' std::vector<Object> ret;\n'
b' const auto& numOutputs = m_trtEngine->getOutputDims().size();\n'
b' std::vector<std::vector<float>> featureVector;\n'
b' Engine<float>::transformOutput(featureVectors, featureVector);\n'
b' ret = PostProcessSegmentation(featureVector, camera_id);\n'
b' return ret;\n'
b' }\n'
b' else {\n'
b' this->_logger.LogError("TENSORRTSEG::DetectObjects", "Error running inference", __FILE__, __LINE__);\n'
b' std::vector<Object> ret;\n'
b' ret.clear();\n'
b' return ret;\n'
b' }\n'
b' }\n'
b' catch (std::exception& e) {\n'
b' this->_logger.LogFatal("TENSORRTSEG::DetectObjects", e.what(), __FILE__, __LINE__);\n'
b' std::vector<Object> ret;\n'
b' ret.clear();\n'
b' return ret;\n'
b' }\n'
b' }\n')
new_det = L(
b' std::vector<Object> TENSORRTSEG::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) {\n'
b' // Phase 1: Preprocess under lock\n'
b' ImageMetadata meta;\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> input;\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' input = Preprocess(inputImage, meta);\n'
b' }\n'
b' if (input.empty()) return {};\n'
b'\n'
b' // Phase 2: Inference -- mutex released; pool dispatches to idle GPU slot\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(input, featureVectors);\n'
b' if (!succ) {\n'
b' this->_logger.LogError("TENSORRTSEG::DetectObjects", "Error running inference", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Phase 3: Postprocess under lock\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' std::vector<std::vector<float>> featureVector;\n'
b' Engine<float>::transformOutput(featureVectors, featureVector);\n'
b' return PostProcessSegmentation(featureVector, camera_id, meta);\n'
b' }\n')
data = replace_once(data, old_det, new_det, 'DetectObjects refactor')
# 5. Preprocess - remove inner lock, add outMeta param
old_pre_sig = L(
b' std::vector<std::vector<cv::cuda::GpuMat>> TENSORRTSEG::Preprocess(const cv::Mat& inputImage) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n')
new_pre_sig = L(
b' std::vector<std::vector<cv::cuda::GpuMat>> TENSORRTSEG::Preprocess(const cv::Mat& inputImage, ImageMetadata& outMeta) {\n')
data = replace_once(data, old_pre_sig, new_pre_sig, 'Preprocess sig + remove lock')
# 6. Replace member writes in Preprocess
old_pw = L(
b' // Set image size parameters\n'
b' m_imgHeight = imgRGB.rows;\n'
b' m_imgWidth = imgRGB.cols;\n'
b' if (m_imgHeight > 0 && m_imgWidth > 0) {\n'
b' m_ratio = 1.f / std::min(inputDims[0].d[2] / static_cast<float>(imgRGB.cols),\n'
b' inputDims[0].d[1] / static_cast<float>(imgRGB.rows));\n')
new_pw = L(
b' // Set image size parameters\n'
b' outMeta.imgHeight = imgRGB.rows;\n'
b' outMeta.imgWidth = imgRGB.cols;\n'
b' if (outMeta.imgHeight > 0 && outMeta.imgWidth > 0) {\n'
b' outMeta.ratio = 1.f / std::min(inputDims[0].d[2] / static_cast<float>(imgRGB.cols),\n'
b' inputDims[0].d[1] / static_cast<float>(imgRGB.rows));\n')
data = replace_once(data, old_pw, new_pw, 'Preprocess member writes')
# 7. Replace error log in Preprocess
old_pe = L(b' this->_logger.LogFatal("TENSORRTCL::Preprocess",\n'
b' "Image height or width is zero after processing (Width: " + std::to_string(m_imgWidth) +\n'
b' ", Height: " + std::to_string(m_imgHeight) + ")",\n')
new_pe = L(b' this->_logger.LogFatal("TENSORRTCL::Preprocess",\n'
b' "Image height or width is zero after processing (Width: " + std::to_string(outMeta.imgWidth) +\n'
b' ", Height: " + std::to_string(outMeta.imgHeight) + ")",\n')
data = replace_once(data, old_pe, new_pe, 'Preprocess error log')
# 8. PostProcessSegmentation:
# 8a. Change signature (remove lock, add meta param)
old_pss_sig = L(
b' std::vector<Object> TENSORRTSEG::PostProcessSegmentation(std::vector<std::vector<float>>& featureVectors, const std::string& camera_id) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n')
new_pss_sig = L(
b' std::vector<Object> TENSORRTSEG::PostProcessSegmentation(std::vector<std::vector<float>>& featureVectors, const std::string& camera_id, const ImageMetadata& meta) {\n')
data = replace_once(data, old_pss_sig, new_pss_sig, 'PostProcessSegmentation sig + remove lock')
# 8b. Fix dead-code bug: flip the if to == (sizes match -> process) so else returns early.
# Also remove the dead inner-if block and its dead return.
# Strategy: replace the opening dead-code block with flipped condition.
old_deadopen = L(
b' if (featureVectors[0].size() != static_cast<size_t>(numChannels) * numAnchors) {\n'
b' std::vector<Object>result;\n'
b' result.clear();\n'
b' return result;\n'
b'\n'
b' if (featureVectors[1].size() != static_cast<size_t>(SEG_CHANNELS) * SEG_H * SEG_W) {\n'
b' std::vector<Object>result;\n'
b' result.clear();\n'
b' return result;\n'
b' }\n'
b'\n')
new_deadopen = L(
b' if (featureVectors[0].size() == static_cast<size_t>(numChannels) * numAnchors &&\n'
b' featureVectors[1].size() == static_cast<size_t>(SEG_CHANNELS) * SEG_H * SEG_W) {\n'
b'\n')
data = replace_once(data, old_deadopen, new_deadopen, 'PostProcessSegmentation dead-code flip')
# 8c. Replace m_ratio/m_imgWidth/m_imgHeight -> meta.* in PostProcessSegmentation
# After steps 6+7, only PostProcessSegmentation body still uses these members.
data = replace_all(data, b'm_ratio', b'meta.ratio', 4, 'meta.ratio (4x in PostProcessSeg)')
data = replace_all(data, b'm_imgWidth', b'meta.imgWidth', 7, 'meta.imgWidth (7x in PostProcessSeg)')
data = replace_all(data, b'm_imgHeight', b'meta.imgHeight', 7, 'meta.imgHeight (7x in PostProcessSeg)')
# 9. Replace DetectObjectsBatch + PreprocessBatch + PostProcessSegmentationBatch bodies
old_batch_all = L(
b' std::vector<Object> TENSORRTSEG::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' std::vector<Object> allObjects;\n'
b'\n'
b' try {\n'
b' // Validate inputs\n'
b' if (inputImages.empty()) {\n'
b' _logger.LogFatal("TENSORRTSEG::DetectObjectsBatch", "Empty input images vector", __FILE__, __LINE__);\n'
b' return allObjects;\n'
b' }\n'
b'\n'
b' _logger.LogInfo("TENSORRTSEG::DetectObjectsBatch",\n'
b' "Processing batch of " + std::to_string(inputImages.size()) + " images",\n'
b' __FILE__, __LINE__);\n'
b'\n'
b' // Preprocess all images in batch\n'
b' const auto inputs = PreprocessBatch(inputImages);\n'
b'\n'
b' if (inputs.empty() || inputs[0].empty()) {\n'
b' _logger.LogFatal("TENSORRTSEG::DetectObjectsBatch", "Preprocessing failed", __FILE__, __LINE__);\n'
b' return allObjects;\n'
b' }\n'
b'\n'
b' // Run batch inference\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(inputs, featureVectors);\n'
b'\n'
b' if (!succ) {\n'
b' _logger.LogError("TENSORRTSEG::DetectObjectsBatch", "Error running inference", __FILE__, __LINE__);\n'
b' return allObjects;\n'
b' }\n'
b'\n'
b' // Process each image\'s results\n'
b' for (size_t batchIdx = 0; batchIdx < featureVectors.size(); ++batchIdx) {\n'
b' // Transform output for this batch element\n'
b' std::vector<std::vector<float>> featureVector;\n'
b'\n'
b' // featureVectors[batchIdx] contains the outputs for this image\n'
b' // For segmentation: [detection_output, segmentation_masks]\n'
b' if (!featureVectors[batchIdx].empty()) {\n'
b' featureVector = featureVectors[batchIdx];\n'
b' }\n'
b'\n'
b' // Get detections for this image\n'
b' std::vector<Object> imageObjects = PostProcessSegmentationBatch(featureVector, camera_id, batchIdx);\n'
b'\n'
b' // Combine all detections into single vector\n'
b' allObjects.insert(allObjects.end(),\n'
b' std::make_move_iterator(imageObjects.begin()),\n'
b' std::make_move_iterator(imageObjects.end()));\n'
b' }\n'
b'\n'
b' _logger.LogInfo("TENSORRTSEG::DetectObjectsBatch",\n'
b' "Batch processing complete. Total detections: " + std::to_string(allObjects.size()),\n'
b' __FILE__, __LINE__);\n'
b'\n'
b' return allObjects;\n'
b' }\n'
b' catch (std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::DetectObjectsBatch", e.what(), __FILE__, __LINE__);\n'
b' return allObjects;\n'
b' }\n'
b' }\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> TENSORRTSEG::PreprocessBatch(const std::vector<cv::Mat>& inputImages) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b'\n'
b' try {\n'
b' if (!_licenseValid) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch", "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' const auto& inputDims = m_trtEngine->getInputDims();\n'
b' const int inputH = inputDims[0].d[1];\n'
b' const int inputW = inputDims[0].d[2];\n'
b'\n'
b' // Store original image dimensions for each image in batch\n'
b' m_batchImgHeights.resize(inputImages.size());\n'
b' m_batchImgWidths.resize(inputImages.size());\n'
b' m_batchRatios.resize(inputImages.size());\n'
b'\n'
b' std::vector<cv::cuda::GpuMat> batchProcessed;\n'
b' batchProcessed.reserve(inputImages.size());\n'
b'\n'
b' cv::cuda::Stream stream;\n'
b'\n'
b' // Process each image\n'
b' for (size_t i = 0; i < inputImages.size(); ++i) {\n'
b' const auto& inputImage = inputImages[i];\n'
b'\n'
b' if (inputImage.empty()) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch",\n'
b' "Empty input image at index " + std::to_string(i), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Upload to GPU\n'
b' cv::cuda::GpuMat img;\n'
b'\n'
b' // Convert grayscale to BGR if needed\n'
b' if (inputImage.channels() == 1) {\n'
b' cv::Mat img3Channel;\n'
b' cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);\n'
b' img.upload(img3Channel, stream);\n'
b' }\n'
b' else {\n'
b' img.upload(inputImage, stream);\n'
b' }\n'
b'\n'
b' // Convert to RGB\n'
b' cv::cuda::GpuMat imgRGB;\n'
b' cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);\n'
b'\n'
b' // Store original dimensions\n'
b' m_batchImgHeights[i] = imgRGB.rows;\n'
b' m_batchImgWidths[i] = imgRGB.cols;\n'
b'\n'
b' if (m_batchImgHeights[i] <= 0 || m_batchImgWidths[i] <= 0) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch",\n'
b' "Image " + std::to_string(i) + " has invalid dimensions (Width: " +\n'
b' std::to_string(m_batchImgWidths[i]) + ", Height: " +\n'
b' std::to_string(m_batchImgHeights[i]) + ")",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Calculate ratio for this image\n'
b' m_batchRatios[i] = 1.f / std::min(inputW / static_cast<float>(imgRGB.cols),\n'
b' inputH / static_cast<float>(imgRGB.rows));\n'
b'\n'
b' // Resize with padding\n'
b' cv::cuda::GpuMat resized = imgRGB;\n'
b' if (resized.rows != inputH || resized.cols != inputW) {\n'
b' resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(imgRGB, inputH, inputW);\n'
b' }\n'
b'\n'
b' batchProcessed.push_back(std::move(resized));\n'
b' }\n'
b'\n'
b' stream.waitForCompletion();\n'
b'\n'
b' // Return as required format\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> inputs;\n'
b' inputs.push_back(std::move(batchProcessed));\n'
b'\n'
b' return inputs;\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b' std::vector<Object> TENSORRTSEG::PostProcessSegmentationBatch(std::vector<std::vector<float>>& featureVectors,const std::string& camera_id,size_t batchIdx) {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b'\n'
b' try {\n'
b' if (!_licenseValid) {\n'
b' _logger.LogFatal("TENSORRTSEG::PostProcessSegmentationBatch", "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' const auto& outputDims = m_trtEngine->getOutputDims();\n'
b'\n'
b' int numChannels = outputDims[0].d[1];\n'
b' int numAnchors = outputDims[0].d[2];\n'
b'\n'
b' const auto numClasses = numChannels - SEG_CHANNELS - 4;\n'
b'\n'
b' // Get batch-specific dimensions\n'
b' float ratio = m_batchRatios[batchIdx];\n'
b' float imgWidth = static_cast<float>(m_batchImgWidths[batchIdx]);\n'
b' float imgHeight = static_cast<float>(m_batchImgHeights[batchIdx]);\n')
new_batch_all = L(
b' std::vector<std::vector<Object>> TENSORRTSEG::DetectObjectsBatch(const std::vector<cv::Mat>& inputImages, const std::string& camera_id) {\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' if (inputImages.empty()) {\n'
b' _logger.LogFatal("TENSORRTSEG::DetectObjectsBatch", "Empty input images vector", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b'\n'
b' // Auto-split if batch exceeds engine capacity\n'
b' const int maxBatch = m_options.maxBatchSize > 0 ? m_options.maxBatchSize : 1;\n'
b' if (static_cast<int>(inputImages.size()) > maxBatch) {\n'
b' const size_t numImages = inputImages.size();\n'
b' std::vector<std::vector<cv::Mat>> chunks;\n'
b' for (size_t start = 0; start < numImages; start += static_cast<size_t>(maxBatch)) {\n'
b' const size_t end = std::min(start + static_cast<size_t>(maxBatch), numImages);\n'
b' chunks.emplace_back(inputImages.begin() + start, inputImages.begin() + end);\n'
b' }\n'
b' std::vector<std::future<std::vector<std::vector<Object>>>> futures;\n'
b' futures.reserve(chunks.size());\n'
b' for (size_t i = 0; i < chunks.size(); ++i) {\n'
b' futures.push_back(std::async(std::launch::async,\n'
b' [this, c = chunks[i], cid = camera_id]() {\n'
b' return DetectObjectsBatch(c, cid);\n'
b' }));\n'
b' }\n'
b' std::vector<std::vector<Object>> allResults;\n'
b' allResults.reserve(numImages);\n'
b' for (auto& fut : futures) {\n'
b' auto chunkResults = fut.get();\n'
b' for (auto& r : chunkResults) allResults.push_back(std::move(r));\n'
b' }\n'
b' return allResults;\n'
b' }\n'
b'\n'
b' _logger.LogInfo("TENSORRTSEG::DetectObjectsBatch",\n'
b' "Processing batch of " + std::to_string(inputImages.size()) + " images",\n'
b' __FILE__, __LINE__);\n'
b'\n'
b' // Phase 1: Preprocess under brief lock\n'
b' BatchMetadata metadata;\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> inputs;\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' inputs = PreprocessBatch(inputImages, metadata);\n'
b' }\n'
b' if (inputs.empty() || inputs[0].empty()) {\n'
b' _logger.LogFatal("TENSORRTSEG::DetectObjectsBatch", "Preprocessing failed", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Phase 2: Inference -- mutex released; pool dispatches to idle GPU slot\n'
b' std::vector<std::vector<std::vector<float>>> featureVectors;\n'
b' auto succ = m_trtEngine->runInference(inputs, featureVectors);\n'
b' if (!succ) {\n'
b' _logger.LogError("TENSORRTSEG::DetectObjectsBatch", "Error running inference", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Phase 3: Parallel postprocessing -- each image is independent\n'
b' const size_t numBatch = featureVectors.size();\n'
b' std::vector<std::vector<Object>> batchDetections(numBatch);\n'
b' std::vector<std::future<std::vector<Object>>> postFutures;\n'
b' postFutures.reserve(numBatch);\n'
b' for (size_t batchIdx = 0; batchIdx < numBatch; ++batchIdx) {\n'
b' const auto& batchOutput = featureVectors[batchIdx];\n'
b' std::vector<std::vector<float>> fv =\n'
b' batchOutput.empty() ? std::vector<std::vector<float>>{} : batchOutput;\n'
b' postFutures.push_back(std::async(std::launch::async,\n'
b' [this, fv = std::move(fv), cid = camera_id,\n'
b' idx = batchIdx, &metadata]() mutable {\n'
b' return PostProcessSegmentationBatch(fv, cid, idx, metadata);\n'
b' }));\n'
b' }\n'
b' for (size_t i = 0; i < numBatch; ++i)\n'
b' batchDetections[i] = postFutures[i].get();\n'
b'\n'
b' _logger.LogInfo("TENSORRTSEG::DetectObjectsBatch",\n'
b' "Batch processing complete. Images: " + std::to_string(numBatch),\n'
b' __FILE__, __LINE__);\n'
b' return batchDetections;\n'
b' }\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> TENSORRTSEG::PreprocessBatch(const std::vector<cv::Mat>& inputImages, BatchMetadata& outMetadata) {\n'
b' try {\n'
b' if (!_licenseValid) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch", "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' const auto& inputDims = m_trtEngine->getInputDims();\n'
b' const int inputH = inputDims[0].d[1];\n'
b' const int inputW = inputDims[0].d[2];\n'
b'\n'
b' // Store original image dimensions for each image in batch\n'
b' outMetadata.imgHeights.resize(inputImages.size());\n'
b' outMetadata.imgWidths.resize(inputImages.size());\n'
b' outMetadata.ratios.resize(inputImages.size());\n'
b'\n'
b' std::vector<cv::cuda::GpuMat> batchProcessed;\n'
b' batchProcessed.reserve(inputImages.size());\n'
b'\n'
b' cv::cuda::Stream stream;\n'
b'\n'
b' // Process each image\n'
b' for (size_t i = 0; i < inputImages.size(); ++i) {\n'
b' const auto& inputImage = inputImages[i];\n'
b'\n'
b' if (inputImage.empty()) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch",\n'
b' "Empty input image at index " + std::to_string(i), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Upload to GPU\n'
b' cv::cuda::GpuMat img;\n'
b'\n'
b' // Convert grayscale to BGR if needed\n'
b' if (inputImage.channels() == 1) {\n'
b' cv::Mat img3Channel;\n'
b' cv::cvtColor(inputImage, img3Channel, cv::COLOR_GRAY2BGR);\n'
b' img.upload(img3Channel, stream);\n'
b' }\n'
b' else {\n'
b' img.upload(inputImage, stream);\n'
b' }\n'
b'\n'
b' // Convert to RGB\n'
b' cv::cuda::GpuMat imgRGB;\n'
b' cv::cuda::cvtColor(img, imgRGB, cv::COLOR_BGR2RGB, 0, stream);\n'
b'\n'
b' // Store original dimensions\n'
b' outMetadata.imgHeights[i] = imgRGB.rows;\n'
b' outMetadata.imgWidths[i] = imgRGB.cols;\n'
b'\n'
b' if (outMetadata.imgHeights[i] <= 0 || outMetadata.imgWidths[i] <= 0) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch",\n'
b' "Image " + std::to_string(i) + " has invalid dimensions (Width: " +\n'
b' std::to_string(outMetadata.imgWidths[i]) + ", Height: " +\n'
b' std::to_string(outMetadata.imgHeights[i]) + ")",\n'
b' __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' // Calculate ratio for this image\n'
b' outMetadata.ratios[i] = 1.f / std::min(inputW / static_cast<float>(imgRGB.cols),\n'
b' inputH / static_cast<float>(imgRGB.rows));\n'
b'\n'
b' // Resize with padding\n'
b' cv::cuda::GpuMat resized = imgRGB;\n'
b' if (resized.rows != inputH || resized.cols != inputW) {\n'
b' resized = Engine<float>::resizeKeepAspectRatioPadRightBottom(imgRGB, inputH, inputW);\n'
b' }\n'
b'\n'
b' batchProcessed.push_back(std::move(resized));\n'
b' }\n'
b'\n'
b' stream.waitForCompletion();\n'
b'\n'
b' // Return as required format\n'
b' std::vector<std::vector<cv::cuda::GpuMat>> inputs;\n'
b' inputs.push_back(std::move(batchProcessed));\n'
b'\n'
b' return inputs;\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::PreprocessBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b' std::vector<Object> TENSORRTSEG::PostProcessSegmentationBatch(std::vector<std::vector<float>>& featureVectors, const std::string& camera_id, size_t batchIdx, const BatchMetadata& metadata) {\n'
b' try {\n'
b' if (!_licenseValid) {\n'
b' _logger.LogFatal("TENSORRTSEG::PostProcessSegmentationBatch", "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b'\n'
b' const auto& outputDims = m_trtEngine->getOutputDims();\n'
b'\n'
b' int numChannels = outputDims[0].d[1];\n'
b' int numAnchors = outputDims[0].d[2];\n'
b'\n'
b' const auto numClasses = numChannels - SEG_CHANNELS - 4;\n'
b'\n'
b' // Get batch-specific dimensions\n'
b' float ratio = metadata.ratios[batchIdx];\n'
b' float imgWidth = static_cast<float>(metadata.imgWidths[batchIdx]);\n'
b' float imgHeight = static_cast<float>(metadata.imgHeights[batchIdx]);\n')
data = replace_once(data, old_batch_all, new_batch_all, 'DetectObjectsBatch+PreprocessBatch+PostProcessSegmentationBatch rewrite')
# 10. Add RunInferencesBatch before namespace close
old_ns_close = L(
b' return objs;\n'
b' }\n'
b' catch (std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::PostProcessSegmentationBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b' }\n')
new_ns_close = L(
b' return objs;\n'
b' }\n'
b' catch (std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::PostProcessSegmentationBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b' std::vector<std::vector<Object>> TENSORRTSEG::RunInferencesBatch(\n'
b' const std::vector<cv::Mat>& inputs, const std::string& camera_id)\n'
b' {\n'
b' {\n'
b' std::lock_guard<std::recursive_mutex> lock(_mutex);\n'
b' if (!_modelLoadValid) {\n'
b' _logger.LogError("TENSORRTSEG::RunInferencesBatch",\n'
b' "Model not loaded", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (!_licenseValid) {\n'
b' _logger.LogError("TENSORRTSEG::RunInferencesBatch",\n'
b' "Invalid license", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (!_isInitialized) {\n'
b' _logger.LogError("TENSORRTSEG::RunInferencesBatch",\n'
b' "Engine not initialized", __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' if (inputs.empty()) return {};\n'
b' }\n'
b' try {\n'
b' return DetectObjectsBatch(inputs, camera_id);\n'
b' }\n'
b' catch (const std::exception& e) {\n'
b' _logger.LogFatal("TENSORRTSEG::RunInferencesBatch", e.what(), __FILE__, __LINE__);\n'
b' return {};\n'
b' }\n'
b' }\n'
b'}\n')
data = replace_once(data, old_ns_close, new_ns_close, 'Add RunInferencesBatch + fix namespace close')
# Summary
print(f'\nOriginal size: {orig_size} bytes')
print(f'New size: {len(data)} bytes')
if errors:
print(f'\n{len(errors)} ERROR(S) -- file NOT written')
sys.exit(1)
else:
with open(CPP, 'wb') as f:
f.write(data)
print('\nOK -- file written')
+5
View File
@@ -0,0 +1,5 @@
// pch.cpp: source file corresponding to the pre-compiled header
#include "pch.h"
// When you are using pre-compiled headers, this source file is necessary for compilation to succeed.

Some files were not shown because too many files have changed in this diff Show More