ANSCENTER/ANSCORE
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CPU/GPU gate and support new ANSALPR using OCR

Browse Source
This commit is contained in:
Tuan Nghia Nguyen
2026-04-12 17:16:16 +10:00
parent 27083a6530
commit 0a8aaed215
30 changed files with 1870 additions and 2166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

843
modules/ANSLPR/ANSALPR_OV.cpp
View File

@@ -1,843 +0,0 @@
#include "ANSLPR_OV.h"
namespace ANSCENTER {
void tryPush(const std::weak_ptr<Worker>& worker, std::shared_ptr<Task>&& task) {
try {
std::shared_ptr<Worker>(worker)->push(task);
} catch (const std::bad_weak_ptr&) {}
}
void fillROIColor(cv::Mat& displayImage, cv::Rect roi, cv::Scalar color, double opacity) {
if (opacity > 0) {
roi = roi & cv::Rect(0, 0, displayImage.cols, displayImage.rows);
cv::Mat textROI = displayImage(roi);
cv::addWeighted(color, opacity, textROI, 1.0 - opacity , 0.0, textROI);
}
}
void putTextOnImage(cv::Mat& displayImage, std::string str, cv::Point p,
cv::HersheyFonts font, double fontScale, cv::Scalar color,
int thickness = 1, cv::Scalar bgcolor = cv::Scalar(),
double opacity = 0) {
int baseline = 0;
cv::Size textSize = cv::getTextSize(str, font, 0.5, 1, &baseline);
fillROIColor(displayImage, cv::Rect(cv::Point(p.x, p.y + baseline),
cv::Point(p.x + textSize.width, p.y - textSize.height)),
bgcolor, opacity);
cv::putText(displayImage, str, p, font, fontScale, color, thickness);
}
Detector::Detector(ov::Core& core,
const std::string& deviceName,
const std::string& xmlPath,
const std::vector<float>& detectionTresholds,
const bool autoResize) :
m_autoResize(autoResize),
m_detectionTresholds{ detectionTresholds }
{
slog::info << "Reading model: " << xmlPath << slog::endl;
std::shared_ptr<ov::Model> model = core.read_model(xmlPath);
logBasicModelInfo(model);
// Check model inputs and outputs
ov::OutputVector inputs = model->inputs();
if (inputs.size() != 1) {
throw std::logic_error("Detector should have only one input");
}
m_detectorInputName = model->input().get_any_name();
ov::Layout modelLayout = ov::layout::get_layout(model->input());
if (modelLayout.empty())
modelLayout = { "NCHW" };
ov::OutputVector outputs = model->outputs();
if (outputs.size() != 1) {
throw std::logic_error("Vehicle Detection network should have only one output");
}
ov::Output<ov::Node> output = outputs[0];
m_detectorOutputName = output.get_any_name();
ov::Shape output_shape = output.get_shape();
if (output_shape.size() != 4) {
throw std::logic_error("Incorrect output dimensions for SSD");
}
if (maxProposalCount != output_shape[2]) {
throw std::logic_error("unexpected ProposalCount");
}
if (objectSize != output_shape[3]) {
throw std::logic_error("Output should have 7 as a last dimension");
}
ov::preprocess::PrePostProcessor ppp(model);
ov::preprocess::InputInfo& inputInfo = ppp.input();
ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor();
// configure desired input type and layout, the
// use preprocessor to convert to actual model input type and layout
inputTensorInfo.set_element_type(ov::element::u8);
inputTensorInfo.set_layout({ "NHWC" });
if (autoResize) {
inputTensorInfo.set_spatial_dynamic_shape();
}
ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model();
inputModelInfo.set_layout(modelLayout);
ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess();
preProcessSteps.convert_layout(modelLayout);
preProcessSteps.convert_element_type(ov::element::f32);
if (autoResize) {
preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
}
model = ppp.build();
slog::info << "Preprocessor configuration: " << slog::endl;
slog::info << ppp << slog::endl;
m_compiled_model = core.compile_model(model, deviceName);
logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "Vehicle And License Plate Detection");
}
ov::InferRequest Detector::createInferRequest() {
return m_compiled_model.create_infer_request();
}
void Detector::setImage(ov::InferRequest& inferRequest, const cv::Mat& img) {
ov::Tensor inputTensor = inferRequest.get_tensor(m_detectorInputName);
ov::Shape shape = inputTensor.get_shape();
if (m_autoResize) {
if (!img.isSubmatrix()) {
// just wrap Mat object with Tensor without additional memory allocation
ov::Tensor frameTensor = wrapMat2Tensor(img);
inferRequest.set_tensor(m_detectorInputName, frameTensor);
}
else {
throw std::logic_error("Sparse matrix are not supported");
}
}
else {
// resize and copy data from image to tensor using OpenCV
resize2tensor(img, inputTensor);
}
}
std::list<Detector::Result> Detector::getResults(ov::InferRequest& inferRequest,
cv::Size upscale,
std::vector<std::string>& rawResults) {
// there is no big difference if InferReq of detector from another device is passed
// because the processing is the same for the same topology
std::list<Result> results;
ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName);
const float* const detections = output_tensor.data<float>();
// pretty much regular SSD post-processing
for (int i = 0; i < maxProposalCount; i++) {
float image_id = detections[i * objectSize + 0]; // in case of batch
if (image_id < 0) { // indicates end of detections
break;
}
size_t label = static_cast<decltype(m_detectionTresholds.size())>(detections[i * objectSize + 1]);
float confidence = detections[i * objectSize + 2];
if (label - 1 < m_detectionTresholds.size() && confidence < m_detectionTresholds[label - 1]) {
continue;
}
cv::Rect rect;
rect.x = static_cast<int>(detections[i * objectSize + 3] * upscale.width);
rect.y = static_cast<int>(detections[i * objectSize + 4] * upscale.height);
rect.width = static_cast<int>(detections[i * objectSize + 5] * upscale.width) - rect.x;
rect.height = static_cast<int>(detections[i * objectSize + 6] * upscale.height) - rect.y;
results.push_back(Result{ label, confidence, rect });
std::ostringstream rawResultsStream;
rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence
<< " (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")";
rawResults.push_back(rawResultsStream.str());
}
return results;
}
VehicleAttributesClassifier::VehicleAttributesClassifier(ov::Core& core,
const std::string& deviceName,
const std::string& xmlPath,
const bool autoResize) :m_autoResize(autoResize)
{
slog::info << "Reading model: " << xmlPath << slog::endl;
std::shared_ptr<ov::Model> model = core.read_model(xmlPath);
logBasicModelInfo(model);
ov::OutputVector inputs = model->inputs();
if (inputs.size() != 1) {
throw std::logic_error("Vehicle Attribs topology should have only one input");
}
m_attributesInputName = model->input().get_any_name();
ov::Layout modelLayout = ov::layout::get_layout(model->input());
if (modelLayout.empty())
modelLayout = { "NCHW" };
ov::OutputVector outputs = model->outputs();
if (outputs.size() != 2) {
throw std::logic_error("Vehicle Attribs Network expects networks having two outputs");
}
// color is the first output
m_outputNameForColor = outputs[0].get_any_name();
// type is the second output.
m_outputNameForType = outputs[1].get_any_name();
ov::preprocess::PrePostProcessor ppp(model);
ov::preprocess::InputInfo& inputInfo = ppp.input();
ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor();
// configure desired input type and layout, the
// use preprocessor to convert to actual model input type and layout
inputTensorInfo.set_element_type(ov::element::u8);
inputTensorInfo.set_layout({ "NHWC" });
if (autoResize) {
inputTensorInfo.set_spatial_dynamic_shape();
}
ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess();
preProcessSteps.convert_layout(modelLayout);
preProcessSteps.convert_element_type(ov::element::f32);
if (autoResize) {
preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
}
ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model();
inputModelInfo.set_layout(modelLayout);
model = ppp.build();
slog::info << "Preprocessor configuration: " << slog::endl;
slog::info << ppp << slog::endl;
m_compiled_model = core.compile_model(model, deviceName);
logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "Vehicle Attributes Recognition");
}
ov::InferRequest VehicleAttributesClassifier::createInferRequest() {
return m_compiled_model.create_infer_request();
}
void VehicleAttributesClassifier::setImage(ov::InferRequest& inferRequest,
const cv::Mat& img,
const cv::Rect vehicleRect)
{
ov::Tensor inputTensor = inferRequest.get_tensor(m_attributesInputName);
ov::Shape shape = inputTensor.get_shape();
if (m_autoResize) {
ov::Tensor frameTensor = wrapMat2Tensor(img);
ov::Coordinate p00({ 0, static_cast<size_t>(vehicleRect.y), static_cast<size_t>(vehicleRect.x), 0 });
ov::Coordinate p01({ 1, static_cast<size_t>(vehicleRect.y + vehicleRect.height), static_cast<size_t>(vehicleRect.x) + vehicleRect.width, 3 });
ov::Tensor roiTensor(frameTensor, p00, p01);
inferRequest.set_tensor(m_attributesInputName, roiTensor);
}
else {
const cv::Mat& vehicleImage = img(vehicleRect);
resize2tensor(vehicleImage, inputTensor);
}
}
std::pair<std::string, std::string> VehicleAttributesClassifier::getResults(ov::InferRequest& inferRequest) {
static const std::string colors[] = {
"white", "gray", "yellow", "red", "green", "blue", "black"
};
static const std::string types[] = {
"car", "van", "truck", "bus"
};
// 7 possible colors for each vehicle and we should select the one with the maximum probability
ov::Tensor colorsTensor = inferRequest.get_tensor(m_outputNameForColor);
const float* colorsValues = colorsTensor.data<float>();
// 4 possible types for each vehicle and we should select the one with the maximum probability
ov::Tensor typesTensor = inferRequest.get_tensor(m_outputNameForType);
const float* typesValues = typesTensor.data<float>();
const auto color_id = std::max_element(colorsValues, colorsValues + 7) - colorsValues;
const auto type_id = std::max_element(typesValues, typesValues + 4) - typesValues;
return std::pair<std::string, std::string>(colors[color_id], types[type_id]);
}
Lpr::Lpr(ov::Core& core,
const std::string& deviceName,
const std::string& xmlPath,
const bool autoResize) :m_autoResize(autoResize)
{
slog::info << "Reading model: " << xmlPath << slog::endl;
std::shared_ptr<ov::Model> model = core.read_model(xmlPath);
logBasicModelInfo(model);
// LPR network should have 2 inputs (and second is just a stub) and one output
// Check inputs
ov::OutputVector inputs = model->inputs();
if (inputs.size() != 1 && inputs.size() != 2) {
throw std::logic_error("LPR should have 1 or 2 inputs");
}
for (auto input : inputs) {
if (input.get_shape().size() == 4) {
m_LprInputName = input.get_any_name();
m_modelLayout = ov::layout::get_layout(input);
if (m_modelLayout.empty())
m_modelLayout = { "NCHW" };
}
// LPR model that converted from Caffe have second a stub input
if (input.get_shape().size() == 2)
m_LprInputSeqName = input.get_any_name();
}
// Check outputs
m_maxSequenceSizePerPlate = 1;
ov::OutputVector outputs = model->outputs();
if (outputs.size() != 1) {
throw std::logic_error("LPR should have 1 output");
}
m_LprOutputName = outputs[0].get_any_name();
for (size_t dim : outputs[0].get_shape()) {
if (dim == 1) {
continue;
}
if (m_maxSequenceSizePerPlate == 1) {
m_maxSequenceSizePerPlate = dim;
}
else {
throw std::logic_error("Every dimension of LPR output except for one must be of size 1");
}
}
ov::preprocess::PrePostProcessor ppp(model);
ov::preprocess::InputInfo& inputInfo = ppp.input(m_LprInputName);
ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor();
// configure desired input type and layout, the
// use preprocessor to convert to actual model input type and layout
inputTensorInfo.set_element_type(ov::element::u8);
inputTensorInfo.set_layout({ "NHWC" });
if (autoResize) {
inputTensorInfo.set_spatial_dynamic_shape();
}
ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess();
preProcessSteps.convert_layout(m_modelLayout);
preProcessSteps.convert_element_type(ov::element::f32);
if (autoResize) {
preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
}
ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model();
inputModelInfo.set_layout(m_modelLayout);
model = ppp.build();
slog::info << "Preprocessor configuration: " << slog::endl;
slog::info << ppp << slog::endl;
m_compiled_model = core.compile_model(model, deviceName);
logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "License Plate Recognition");
}
ov::InferRequest Lpr::createInferRequest() {
return m_compiled_model.create_infer_request();
}
void Lpr::setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect plateRect) {
ov::Tensor inputTensor = inferRequest.get_tensor(m_LprInputName);
ov::Shape shape = inputTensor.get_shape();
if ((shape.size() == 4) && m_autoResize) {
// autoResize is set
ov::Tensor frameTensor = wrapMat2Tensor(img);
ov::Coordinate p00({ 0, static_cast<size_t>(plateRect.y), static_cast<size_t>(plateRect.x), 0 });
ov::Coordinate p01({ 1, static_cast<size_t>(plateRect.y + plateRect.height), static_cast<size_t>(plateRect.x + plateRect.width), 3 });
ov::Tensor roiTensor(frameTensor, p00, p01);
inferRequest.set_tensor(m_LprInputName, roiTensor);
}
else {
const cv::Mat& vehicleImage = img(plateRect);
resize2tensor(vehicleImage, inputTensor);
}
if (m_LprInputSeqName != "") {
ov::Tensor inputSeqTensor = inferRequest.get_tensor(m_LprInputSeqName);
float* data = inputSeqTensor.data<float>();
std::fill(data, data + inputSeqTensor.get_shape()[0], 1.0f);
}
}
std::string Lpr::getResults(ov::InferRequest& inferRequest) {
static const char* const items[] = {
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
"", "", "", "",
"", "", "", "",
"", "", "", "",
"", "", "", "",
"", "", "", "",
"", "", "", "",
"", "", "", "",
"", "", "", "",
"", "",
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J",
"K", "L", "M", "N", "O", "P", "Q", "R", "S", "T",
"U", "V", "W", "X", "Y", "Z"
};
std::string result;
result.reserve(14u + 6u); // the longest province name + 6 plate signs
ov::Tensor lprOutputTensor = inferRequest.get_tensor(m_LprOutputName);
ov::element::Type precision = lprOutputTensor.get_element_type();
// up to 88 items per license plate, ended with "-1"
switch (precision) {
case ov::element::i32:
{
const auto data = lprOutputTensor.data<int32_t>();
for (int i = 0; i < m_maxSequenceSizePerPlate; i++) {
int32_t val = data[i];
if (val == -1) {
break;
}
result += items[val];
}
}
break;
case ov::element::f32:
{
const auto data = lprOutputTensor.data<float>();
for (int i = 0; i < m_maxSequenceSizePerPlate; i++) {
int32_t val = int32_t(data[i]);
if (val == -1) {
break;
}
result += items[val];
}
}
break;
default:
throw std::logic_error("Not expected output blob precision");
break;
}
return result;
}
// Utilities
ReborningVideoFrame::~ReborningVideoFrame() {
try {
const std::shared_ptr<Worker>& worker = std::shared_ptr<Worker>(context.readersContext.readersWorker);
context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock();
const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID];
context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock();
std::shared_ptr<ReborningVideoFrame> reborn = std::make_shared<ReborningVideoFrame>(context, sourceID, frameId, frame);
worker->push(std::make_shared<Reader>(reborn));
}
catch (const std::bad_weak_ptr&) {}
}
void ResAggregator::process() {
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
context.freeDetectionInfersCount += context.detectorsInfers.inferRequests.lockedSize();
context.frameCounter++;
context.boxesAndDescrs = boxesAndDescrs;
try {
std::shared_ptr<Worker>(context.resAggregatorsWorker)->stop();
}
catch (const std::bad_weak_ptr&) {}
}
bool DetectionsProcessor::isReady() {
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
if (requireGettingNumberOfDetections) {
classifiersAggregator = std::make_shared<ClassifiersAggregator>(sharedVideoFrame);
std::list<Detector::Result> results;
results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections);
for (Detector::Result result : results) {
switch (result.label) {
case 1:// Vehicle
{
vehicleRects.emplace_back(result.location & cv::Rect{ cv::Point(0, 0), sharedVideoFrame->frame.size() });
break;
}
case 2:// License Plate
{
// expanding a bounding box a bit, better for the license plate recognition
result.location.x -= 5;
result.location.y -= 5;
result.location.width += 10;
result.location.height += 10;
plateRects.emplace_back(result.location & cv::Rect{ cv::Point(0, 0), sharedVideoFrame->frame.size() });
break;
}
default:
throw std::runtime_error("Unexpected detection results"); // must never happen
break;
}
}
context.detectorsInfers.inferRequests.lockedPushBack(*inferRequest);
requireGettingNumberOfDetections = false;
}
if ((vehicleRects.empty()) && (plateRects.empty())) {
return true;
}
else {
InferRequestsContainer& attributesInfers = context.attributesInfers;
attributesInfers.inferRequests.mutex.lock();
const std::size_t numberOfAttributesInferRequestsAcquired = std::min(vehicleRects.size(), attributesInfers.inferRequests.container.size());
reservedAttributesRequests.assign(attributesInfers.inferRequests.container.end() - numberOfAttributesInferRequestsAcquired,attributesInfers.inferRequests.container.end());
attributesInfers.inferRequests.container.erase(attributesInfers.inferRequests.container.end() - numberOfAttributesInferRequestsAcquired,attributesInfers.inferRequests.container.end());
attributesInfers.inferRequests.mutex.unlock();
InferRequestsContainer& platesInfers = context.platesInfers;
platesInfers.inferRequests.mutex.lock();
const std::size_t numberOfLprInferRequestsAcquired = std::min(plateRects.size(), platesInfers.inferRequests.container.size());
reservedLprRequests.assign(platesInfers.inferRequests.container.end() - numberOfLprInferRequestsAcquired, platesInfers.inferRequests.container.end());
platesInfers.inferRequests.container.erase(platesInfers.inferRequests.container.end() - numberOfLprInferRequestsAcquired,platesInfers.inferRequests.container.end());
platesInfers.inferRequests.mutex.unlock();
return numberOfAttributesInferRequestsAcquired || numberOfLprInferRequestsAcquired;
}
}
void DetectionsProcessor::process() {
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
auto vehicleRectsIt = vehicleRects.begin();
for (auto attributesRequestIt = reservedAttributesRequests.begin(); attributesRequestIt != reservedAttributesRequests.end();
vehicleRectsIt++, attributesRequestIt++) {
const cv::Rect vehicleRect = *vehicleRectsIt;
ov::InferRequest& attributesRequest = *attributesRequestIt;
context.detectionsProcessorsContext.vehicleAttributesClassifier.setImage(attributesRequest, sharedVideoFrame->frame, vehicleRect);
attributesRequest.set_callback(
std::bind(
[](std::shared_ptr<ClassifiersAggregator> classifiersAggregator,
ov::InferRequest& attributesRequest,
cv::Rect rect,
Context& context) {
attributesRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object
const std::pair<std::string, std::string>& attributes =context.detectionsProcessorsContext.vehicleAttributesClassifier.getResults(attributesRequest);
if (((classifiersAggregator->sharedVideoFrame->frameId == 0 && !context.isVideo) || context.isVideo)) {
classifiersAggregator->rawAttributes.lockedPushBack("Vehicle Attributes results:" + attributes.first + ';' + attributes.second);
}
classifiersAggregator->push(BboxAndDescr{ BboxAndDescr::ObjectType::VEHICLE, rect, attributes.first + ' ' + attributes.second });
context.attributesInfers.inferRequests.lockedPushBack(attributesRequest);
},
classifiersAggregator,
std::ref(attributesRequest),
vehicleRect,
std::ref(context)));
attributesRequest.start_async();
}
vehicleRects.erase(vehicleRects.begin(), vehicleRectsIt);
auto plateRectsIt = plateRects.begin();
for (auto lprRequestsIt = reservedLprRequests.begin(); lprRequestsIt != reservedLprRequests.end(); plateRectsIt++, lprRequestsIt++) {
const cv::Rect plateRect = *plateRectsIt;
ov::InferRequest& lprRequest = *lprRequestsIt;
context.detectionsProcessorsContext.lpr.setImage(lprRequest, sharedVideoFrame->frame, plateRect);
lprRequest.set_callback(
std::bind(
[](std::shared_ptr<ClassifiersAggregator> classifiersAggregator,
ov::InferRequest& lprRequest,
cv::Rect rect,
Context& context) {
lprRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object
std::string result = context.detectionsProcessorsContext.lpr.getResults(lprRequest);
if (((classifiersAggregator->sharedVideoFrame->frameId == 0 && !context.isVideo) || context.isVideo)) {
classifiersAggregator->rawDecodedPlates.lockedPushBack("License Plate Recognition results:" + result);
}
classifiersAggregator->push(BboxAndDescr{ BboxAndDescr::ObjectType::PLATE, rect, std::move(result) });
context.platesInfers.inferRequests.lockedPushBack(lprRequest);
}, classifiersAggregator,
std::ref(lprRequest),
plateRect,
std::ref(context)));
lprRequest.start_async();
}
plateRects.erase(plateRects.begin(), plateRectsIt);
if (!vehicleRects.empty() || !plateRects.empty()) {
tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker,
std::make_shared<DetectionsProcessor>(sharedVideoFrame, std::move(classifiersAggregator), std::move(vehicleRects), std::move(plateRects)));
}
}
bool InferTask::isReady() {
InferRequestsContainer& detectorsInfers = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context.detectorsInfers;
if (detectorsInfers.inferRequests.container.empty()) {
return false;
}
else {
detectorsInfers.inferRequests.mutex.lock();
if (detectorsInfers.inferRequests.container.empty()) {
detectorsInfers.inferRequests.mutex.unlock();
return false;
}
else {
return true; // process() will unlock the mutex
}
}
}
void InferTask::process() {
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
InferRequestsContainer& detectorsInfers = context.detectorsInfers;
std::reference_wrapper<ov::InferRequest> inferRequest = detectorsInfers.inferRequests.container.back();
detectorsInfers.inferRequests.container.pop_back();
detectorsInfers.inferRequests.mutex.unlock();
context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame);
inferRequest.get().set_callback(
std::bind(
[](VideoFrame::Ptr sharedVideoFrame,
ov::InferRequest& inferRequest,
Context& context) {
inferRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object
tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker,
std::make_shared<DetectionsProcessor>(sharedVideoFrame, &inferRequest));
}, sharedVideoFrame,
inferRequest,
std::ref(context)));
inferRequest.get().start_async();
// do not push as callback does it
}
bool Reader::isReady() {
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].lock();
if (context.readersContext.lastCapturedFrameIds[sharedVideoFrame->sourceID] + 1 == sharedVideoFrame->frameId) {
return true;
}
else {
context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].unlock();
return false;
}
}
void Reader::process() {
unsigned sourceID = sharedVideoFrame->sourceID;
sharedVideoFrame->timestamp = std::chrono::steady_clock::now();
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
const std::vector<std::shared_ptr<InputChannel>>& inputChannels = context.readersContext.inputChannels;
if (inputChannels[sourceID]->read(sharedVideoFrame->frame)) {
context.readersContext.lastCapturedFrameIds[sourceID]++;
context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock();
tryPush(context.inferTasksContext.inferTasksWorker, std::make_shared<InferTask>(sharedVideoFrame));
}
else {
context.readersContext.lastCapturedFrameIds[sourceID]++;
context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock();
try {
std::shared_ptr<Worker>(context.resAggregatorsWorker)->stop();
}
catch (const std::bad_weak_ptr&) {}
}
}
/// <summary>
/// Main class
/// </summary>
ANSALPR_OV::ANSALPR_OV() {};
ANSALPR_OV::~ANSALPR_OV() {
if (_detector == nullptr) {
delete _detector;
_detector = nullptr;
}
if (_vehicleAttributesClassifier == nullptr) {
delete _vehicleAttributesClassifier;
_vehicleAttributesClassifier = nullptr;
}
if (_lpr == nullptr) {
delete _lpr;
_lpr = nullptr;
}
};
bool ANSALPR_OV::Destroy() {
if (_detector == nullptr) {
delete _detector;
_detector = nullptr;
}
if (_vehicleAttributesClassifier == nullptr) {
delete _vehicleAttributesClassifier;
_vehicleAttributesClassifier = nullptr;
}
if (_lpr == nullptr) {
delete _lpr;
_lpr = nullptr;
}
return true;
};
bool ANSALPR_OV::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
_licenseKey = licenseKey;
_licenseValid = false;
CheckLicense();
if (!_licenseValid) {
this->_logger->LogError("ANSALPR_OV::Initialize.", "License is not valid.", __FILE__, __LINE__);
return false;
}
// Extract model folder
// 0. Check if the modelZipFilePath exist?
if (!FileExist(modelZipFilePath)) {
this->_logger->LogFatal("ANSALPR_OV::Initialize", "Model zip file is not exist", __FILE__, __LINE__);
}
// 1. Unzip model zip file to a special location with folder name as model file (and version)
std::string outputFolder;
std::vector<std::string> passwordArray;
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
passwordArray.push_back("AnsDemoModels20@!");
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
size_t vectorSize = passwordArray.size();
for (size_t i = 0; i < vectorSize; i++) {
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
break; // Break the loop when the condition is met.
}
// 2. Check if the outputFolder exist
if (!FolderExist(_modelFolder)) {
this->_logger->LogError("ANSALPR_OV::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
return false; // That means the model file is not exist or the password is not correct
}
_vehicleLPModel = CreateFilePath(_modelFolder, "vehiclelp.xml");
_vehicleAtModel = CreateFilePath(_modelFolder, "vehicle.xml");
_lprModel = CreateFilePath(_modelFolder, "lpr.xml");
ov::Core core;
int FLAGS_nthreads = 0;
std::set<std::string> devices;
for (const std::string& netDevices : { "GPU", "GPU", "GPU" }) {
if (netDevices.empty()) {
continue;
}
for (const std::string& device : parseDevices(netDevices)) {
devices.insert(device);
}
}
std::map<std::string, int32_t> device_nstreams = parseValuePerDevice(devices, "");
for (const std::string& device : devices) {
if ("CPU" == device) {
if (FLAGS_nthreads != 0) {
core.set_property("CPU", ov::inference_num_threads(FLAGS_nthreads));
}
//core.set_property("CPU", ov::affinity(ov::Affinity::NONE));
core.set_property("CPU", ov::streams::num((device_nstreams.count("CPU") > 0 ? ov::streams::Num(device_nstreams["CPU"]) : ov::streams::AUTO)));
device_nstreams["CPU"] = core.get_property("CPU", ov::streams::num);
}
if ("GPU" == device) {
core.set_property("GPU", ov::streams::num(device_nstreams.count("GPU") > 0 ? ov::streams::Num(device_nstreams["GPU"]) : ov::streams::AUTO));
device_nstreams["GPU"] = core.get_property("GPU", ov::streams::num);
if (devices.end() != devices.find("CPU")) {
core.set_property("GPU", ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel(1)));
}
}
}
double FLAGS_t = 0.5;
if(FileExist(_vehicleLPModel))_detector = new Detector(core, "GPU", _vehicleLPModel, { static_cast<float>(FLAGS_t), static_cast<float>(FLAGS_t) }, false);
else {
this->_logger->LogFatal("ANSALPR_OV::Initialize", _vehicleLPModel, __FILE__, __LINE__);
}
if(FileExist(_vehicleAtModel))_vehicleAttributesClassifier = new VehicleAttributesClassifier(core, "GPU", _vehicleAtModel, false);
else {
this->_logger->LogFatal("ANSALPR_OV::Initialize", _vehicleAtModel, __FILE__, __LINE__);
}
if(FileExist(_lprModel)) _lpr = new Lpr(core, "CPU", _lprModel, false);
else {
this->_logger->LogFatal("ANSALPR_OV::Initialize", _lprModel, __FILE__, __LINE__);
}
if (FileExist(_vehicleLPModel) &&
FileExist(_vehicleAtModel) &&
FileExist(_lprModel))return true;
else return false;
}
catch (std::exception& e) {
this->_logger->LogFatal("ANSALPR_OV::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
};
bool ANSALPR_OV::Inference(const cv::Mat& input, std::string& lprResult) {
cv::Mat frame = input.clone();
std::shared_ptr<IInputSource> inputSource = std::make_shared<ImageSource>(frame, true);
std::vector<std::shared_ptr<InputChannel>> _inputChannels;
_inputChannels.push_back(InputChannel::create(inputSource));
unsigned nireq = 1;
bool isVideo = false;
std::size_t nclassifiersireq{ 0 };
std::size_t nrecognizersireq{ 0 };
nclassifiersireq = nireq * 3;
nrecognizersireq = nireq * 3;
Context context = { _inputChannels,
*_detector,
*_vehicleAttributesClassifier,
*_lpr,
2,
nireq,
isVideo,
nclassifiersireq,
nrecognizersireq };
std::shared_ptr<Worker> worker = std::make_shared<Worker>(2);
context.readersContext.readersWorker = worker;
context.inferTasksContext.inferTasksWorker = worker;
context.detectionsProcessorsContext.detectionsProcessorsWorker = worker;
context.resAggregatorsWorker = worker;
for (unsigned sourceID = 0; sourceID < _inputChannels.size(); sourceID++) {
VideoFrame::Ptr sharedVideoFrame = std::make_shared<ReborningVideoFrame>(context, sourceID, 0);
worker->push(std::make_shared<Reader>(sharedVideoFrame));
}
// Running
worker->runThreads();
worker->threadFunc();
worker->join();
std::list<BboxAndDescr> boxesAndDescrs = context.boxesAndDescrs;
std::vector<ALPRObject> output;
output.clear();
for(const BboxAndDescr boxesAndDescr: boxesAndDescrs)
{
if (boxesAndDescr.objectType == ANSCENTER::BboxAndDescr::ObjectType::PLATE) {
ALPRObject result;
result.classId = 0;
result.className = boxesAndDescr.descr;
result.confidence = 1.0;
result.box = boxesAndDescr.rect;
output.push_back(result);
}
}
lprResult = VectorDetectionToJsonString(output);
return true;
};
bool ANSALPR_OV::Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult) {
return true;
}
std::string ANSALPR_OV::VectorDetectionToJsonString(const std::vector<ALPRObject>& dets) {
boost::property_tree::ptree root;
boost::property_tree::ptree detectedObjects;
for (int i = 0; i < dets.size(); i++) {
boost::property_tree::ptree detectedNode;
detectedNode.put("class_id", dets[i].classId);
detectedNode.put("class_name", dets[i].className);
detectedNode.put("prob", dets[i].confidence);
detectedNode.put("x", dets[i].box.x);
detectedNode.put("y", dets[i].box.y);
detectedNode.put("width", dets[i].box.width);
detectedNode.put("height", dets[i].box.height);
detectedNode.put("mask", "");//Todo: convert masks to mask with comma seperated dets[i].mask);
detectedNode.put("extra_info", "");
// we might add masks into this using comma seperated string
detectedObjects.push_back(std::make_pair("", detectedNode));
}
root.add_child("results", detectedObjects);
std::ostringstream stream;
boost::property_tree::write_json(stream, root, false);
std::string trackingResult = stream.str();
return trackingResult;
}
}

6
modules/ANSLPR/ANSLPR.h
View File

@@ -119,6 +119,9 @@ namespace ANSCENTER
void SetALPRCheckerEnabled(bool enable) { _enableALPRChecker = enable; }
bool IsALPRCheckerEnabled() const { return _enableALPRChecker; }
virtual void SetCountry(Country country) { _country = country; }
Country GetCountry() const { return _country; }
[[nodiscard]] virtual bool Destroy() = 0;
[[nodiscard]] static std::vector<cv::Rect> GetBoundingBoxes(const std::string& strBBoxes);
[[nodiscard]] static std::string PolygonToString(const std::vector<cv::Point2f>& polygon);
@@ -172,6 +175,9 @@ extern "C" ANSLPR_API int ANSALPR_GetFormats(ANSCENTER::ANSALPR** Handle, LS
// ALPRChecker: 1 = enabled (full-frame auto-detected), 0 = disabled (raw OCR)
extern "C" ANSLPR_API int ANSALPR_SetALPRCheckerEnabled(ANSCENTER::ANSALPR** Handle, int enable);
// Country: 0=VIETNAM, 1=CHINA, 2=AUSTRALIA, 3=USA, 4=INDONESIA, 5=JAPAN
extern "C" ANSLPR_API int ANSALPR_SetCountry(ANSCENTER::ANSALPR** Handle, int country);
// Unicode conversion utilities for LabVIEW wrapper classes
extern "C" ANSLPR_API int ANSLPR_ConvertUTF8ToUTF16LE(const char* utf8Str, LStrHandle result, int includeBOM = 1);
extern "C" ANSLPR_API int ANSLPR_ConvertUTF16LEToUTF8(const unsigned char* utf16leBytes, int byteLen, LStrHandle result);

665
modules/ANSLPR/ANSLPR_OCR.cpp Normal file
View File

@@ -0,0 +1,665 @@
#include "ANSLPR_OCR.h"
#include "ANSONNXYOLO.h"
#include "ANSOnnxOCR.h"
#include "ANSOCRBase.h"
#include <json.hpp>
#include <algorithm>
#include <chrono>
// ---------------------------------------------------------------------------
// SEH wrapper for loading ONNX models — identical to the one in ANSLPR_OD.cpp
// ---------------------------------------------------------------------------
static void WriteEventLog(const char* message, WORD eventType = EVENTLOG_INFORMATION_TYPE) {
static HANDLE hEventLog = RegisterEventSourceA(NULL, "ANSLogger");
if (hEventLog) {
const char* msgs[1] = { message };
ReportEventA(hEventLog, eventType, 0, 0, NULL, 1, 0, msgs, NULL);
}
OutputDebugStringA(message);
OutputDebugStringA("\n");
}
struct LoadOnnxParams_OCR {
const std::string* licenseKey;
ANSCENTER::ModelConfig* config;
const std::string* modelFolder;
const char* modelName;
const char* classFile;
std::string* labels;
std::unique_ptr<ANSCENTER::ANSODBase>* detector;
bool enableTracker;
bool disableStabilization;
};
static bool LoadOnnxModel_OCR_Impl(const LoadOnnxParams_OCR& p) {
try {
auto onnxyolo = std::make_unique<ANSCENTER::ANSONNXYOLO>();
bool ok = onnxyolo->LoadModelFromFolder(
*p.licenseKey, *p.config, p.modelName, p.classFile,
*p.modelFolder, *p.labels);
if (!ok) {
return false;
}
if (p.enableTracker) {
onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, true);
} else {
onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, false);
}
if (p.disableStabilization) {
onnxyolo->SetStabilization(false);
}
*p.detector = std::move(onnxyolo);
return true;
}
catch (...) {
p.detector->reset();
return false;
}
}
static bool LoadOnnxModel_OCR_SEH(const LoadOnnxParams_OCR& p, DWORD* outCode) {
*outCode = 0;
__try {
return LoadOnnxModel_OCR_Impl(p);
}
__except (EXCEPTION_EXECUTE_HANDLER) {
*outCode = GetExceptionCode();
return false;
}
}
namespace ANSCENTER
{
ANSALPR_OCR::ANSALPR_OCR() {
engineType = EngineType::CPU;
}
ANSALPR_OCR::~ANSALPR_OCR() {
try {
Destroy();
}
catch (...) {}
}
bool ANSALPR_OCR::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath,
const std::string& modelZipPassword, double detectorThreshold, double ocrThreshold, double colourThreshold) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
_licenseKey = licenseKey;
_licenseValid = false;
_detectorThreshold = detectorThreshold;
_ocrThreshold = ocrThreshold;
_colorThreshold = colourThreshold;
_country = Country::JAPAN; // Default to JAPAN for OCR-based ALPR
CheckLicense();
if (!_licenseValid) {
this->_logger.LogError("ANSALPR_OCR::Initialize", "License is not valid.", __FILE__, __LINE__);
return false;
}
// Extract model folder
if (!FileExist(modelZipFilePath)) {
this->_logger.LogFatal("ANSALPR_OCR::Initialize", "Model zip file does not exist: " + modelZipFilePath, __FILE__, __LINE__);
return false;
}
this->_logger.LogInfo("ANSALPR_OCR::Initialize", "Model zip file found: " + modelZipFilePath, __FILE__, __LINE__);
// Unzip model zip file
std::vector<std::string> passwordArray;
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
passwordArray.push_back("AnsDemoModels20@!");
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
for (size_t i = 0; i < passwordArray.size(); i++) {
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
break;
}
if (!FolderExist(_modelFolder)) {
this->_logger.LogError("ANSALPR_OCR::Initialize", "Output model folder does not exist: " + _modelFolder, __FILE__, __LINE__);
return false;
}
// Check country from country.txt
std::string countryFile = CreateFilePath(_modelFolder, "country.txt");
if (FileExist(countryFile)) {
std::ifstream infile(countryFile);
std::string countryStr;
std::getline(infile, countryStr);
infile.close();
if (countryStr == "0") _country = Country::VIETNAM;
else if (countryStr == "1") _country = Country::CHINA;
else if (countryStr == "2") _country = Country::AUSTRALIA;
else if (countryStr == "3") _country = Country::USA;
else if (countryStr == "4") _country = Country::INDONESIA;
else if (countryStr == "5") _country = Country::JAPAN;
else _country = Country::JAPAN; // Default for OCR mode
}
// Store the original model zip path — the OCR models (ansocrdec.onnx,
// ansocrcls.onnx, ansocrrec.onnx, dict_ch.txt) are bundled inside the
// same ALPR model zip, so we reuse it for ANSONNXOCR initialization.
_modelZipFilePath = modelZipFilePath;
// Initialize ALPRChecker
alprChecker.Init(MAX_ALPR_FRAME);
_lpColourModelConfig.detectionScoreThreshold = _colorThreshold;
_lpdmodelConfig.detectionScoreThreshold = _detectorThreshold;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSALPR_OCR::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSALPR_OCR::LoadEngine() {
std::lock_guard<std::recursive_mutex> lock(_mutex);
try {
WriteEventLog("ANSALPR_OCR::LoadEngine: Step 1 - Starting engine load");
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 1: Starting engine load", __FILE__, __LINE__);
// Detect hardware
_lpdmodelConfig.detectionScoreThreshold = _detectorThreshold;
_lpColourModelConfig.detectionScoreThreshold = _colorThreshold;
if (_lpdmodelConfig.detectionScoreThreshold < 0.25) _lpdmodelConfig.detectionScoreThreshold = 0.25;
if (_lpdmodelConfig.detectionScoreThreshold > 0.95) _lpdmodelConfig.detectionScoreThreshold = 0.95;
engineType = ANSLicenseHelper::CheckHardwareInformation();
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Detected engine type: " + std::to_string(static_cast<int>(engineType)), __FILE__, __LINE__);
float confThreshold = 0.5f;
float MNSThreshold = 0.5f;
_lpdmodelConfig.modelConfThreshold = confThreshold;
_lpdmodelConfig.modelMNSThreshold = MNSThreshold;
_lpColourModelConfig.modelConfThreshold = confThreshold;
_lpColourModelConfig.modelMNSThreshold = MNSThreshold;
std::string lprModel = CreateFilePath(_modelFolder, "lpd.onnx");
std::string colorModel = CreateFilePath(_modelFolder, "lpc.onnx");
bool valid = false;
// ── Step 2: Load LP detector with ONNX Runtime ───────────────
if (FileExist(lprModel)) {
WriteEventLog("ANSALPR_OCR::LoadEngine: Step 2 - Loading LP detector with ONNX Runtime");
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 2: Loading LP detector with ONNX Runtime", __FILE__, __LINE__);
_lpdmodelConfig.detectionType = DetectionType::DETECTION;
_lpdmodelConfig.modelType = ModelType::ONNXYOLO;
std::string _lprClasses;
{
LoadOnnxParams_OCR p{};
p.licenseKey = &_licenseKey;
p.config = &_lpdmodelConfig;
p.modelFolder = &_modelFolder;
p.modelName = "lpd";
p.classFile = "lpd.names";
p.labels = &_lprClasses;
p.detector = &_lpDetector;
p.enableTracker = true;
p.disableStabilization = true;
DWORD sehCode = 0;
bool lpSuccess = LoadOnnxModel_OCR_SEH(p, &sehCode);
if (sehCode != 0) {
char buf[256];
snprintf(buf, sizeof(buf),
"ANSALPR_OCR::LoadEngine: Step 2 LPD SEH exception 0x%08X — LP detector disabled", sehCode);
WriteEventLog(buf, EVENTLOG_ERROR_TYPE);
this->_logger.LogFatal("ANSALPR_OCR::LoadEngine",
"Step 2: LP detector crashed (SEH). LP detector disabled.", __FILE__, __LINE__);
if (_lpDetector) _lpDetector.reset();
}
else if (!lpSuccess) {
this->_logger.LogError("ANSALPR_OCR::LoadEngine",
"Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__);
if (_lpDetector) _lpDetector.reset();
}
}
}
if (!_lpDetector) {
this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", "LP detector failed to load. Cannot proceed.", __FILE__, __LINE__);
_isInitialized = false;
return false;
}
// ── Step 3: Load OCR engine (ANSONNXOCR) ─────────────────────
// The OCR models (ansocrdec.onnx, ansocrcls.onnx, ansocrrec.onnx,
// dict_ch.txt) are bundled inside the same ALPR model zip, so we
// pass the original ALPR zip path to ANSONNXOCR::Initialize.
// ANSOCRBase::Initialize will extract it (no-op if already done)
// and discover the OCR model files in the extracted folder.
WriteEventLog("ANSALPR_OCR::LoadEngine: Step 3 - Loading OCR engine (ANSONNXOCR)");
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 3: Loading OCR engine (ANSONNXOCR)", __FILE__, __LINE__);
// Verify OCR model files exist in the already-extracted folder
std::string ocrDetModel = CreateFilePath(_modelFolder, "ansocrdec.onnx");
std::string ocrRecModel = CreateFilePath(_modelFolder, "ansocrrec.onnx");
if (!FileExist(ocrDetModel) || !FileExist(ocrRecModel)) {
this->_logger.LogFatal("ANSALPR_OCR::LoadEngine",
"OCR model files not found in model folder: " + _modelFolder +
" (expected ansocrdec.onnx, ansocrrec.onnx)", __FILE__, __LINE__);
_isInitialized = false;
return false;
}
_ocrEngine = std::make_unique<ANSONNXOCR>();
// Determine OCR language based on country
OCRLanguage ocrLang = OCRLanguage::ENGLISH;
switch (_country) {
case Country::JAPAN: ocrLang = OCRLanguage::JAPANESE; break;
case Country::CHINA: ocrLang = OCRLanguage::CHINESE; break;
case Country::VIETNAM: ocrLang = OCRLanguage::ENGLISH; break;
case Country::AUSTRALIA: ocrLang = OCRLanguage::ENGLISH; break;
case Country::USA: ocrLang = OCRLanguage::ENGLISH; break;
case Country::INDONESIA: ocrLang = OCRLanguage::ENGLISH; break;
default: ocrLang = OCRLanguage::ENGLISH; break;
}
OCRModelConfig ocrModelConfig;
ocrModelConfig.ocrLanguage = ocrLang;
ocrModelConfig.useDetector = true;
ocrModelConfig.useRecognizer = true;
ocrModelConfig.useCLS = true;
ocrModelConfig.useLayout = false;
ocrModelConfig.useTable = false;
ocrModelConfig.useTensorRT = false;
ocrModelConfig.enableMKLDNN = false;
ocrModelConfig.useDilation = true;
ocrModelConfig.useAngleCLS = false;
ocrModelConfig.gpuId = 0;
ocrModelConfig.detectionDBThreshold = 0.5;
ocrModelConfig.detectionBoxThreshold = 0.3;
ocrModelConfig.detectionDBUnclipRatio = 1.2;
ocrModelConfig.clsThreshold = 0.9;
ocrModelConfig.limitSideLen = 2560;
// Pass the original ALPR model zip path — ANSOCRBase::Initialize
// will extract it to the same folder (already done, so extraction
// is a no-op) and set up ansocrdec.onnx / ansocrcls.onnx /
// ansocrrec.onnx / dict_ch.txt paths automatically.
bool ocrSuccess = _ocrEngine->Initialize(_licenseKey, ocrModelConfig, _modelZipFilePath, "", 0);
if (!ocrSuccess) {
this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", "Failed to initialize OCR engine (ANSONNXOCR).", __FILE__, __LINE__);
_ocrEngine.reset();
_isInitialized = false;
return false;
}
// Set ALPR mode and country on the OCR engine
_ocrEngine->SetOCRMode(OCRMode::OCR_ALPR);
_ocrEngine->SetCountry(_country);
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 3: OCR engine loaded successfully.", __FILE__, __LINE__);
// ── Step 4: Load colour classifier (optional) ────────────────
if (FileExist(colorModel) && (_lpColourModelConfig.detectionScoreThreshold > 0)) {
WriteEventLog("ANSALPR_OCR::LoadEngine: Step 4 - Loading colour classifier with ONNX Runtime");
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 4: Loading colour classifier with ONNX Runtime", __FILE__, __LINE__);
_lpColourModelConfig.detectionType = DetectionType::CLASSIFICATION;
_lpColourModelConfig.modelType = ModelType::ONNXYOLO;
{
LoadOnnxParams_OCR p{};
p.licenseKey = &_licenseKey;
p.config = &_lpColourModelConfig;
p.modelFolder = &_modelFolder;
p.modelName = "lpc";
p.classFile = "lpc.names";
p.labels = &_lpColourLabels;
p.detector = &_lpColourDetector;
p.enableTracker = false;
p.disableStabilization = false;
DWORD sehCode = 0;
bool colourSuccess = LoadOnnxModel_OCR_SEH(p, &sehCode);
if (sehCode != 0) {
char buf[256];
snprintf(buf, sizeof(buf),
"ANSALPR_OCR::LoadEngine: Step 4 LPC SEH exception 0x%08X — colour detection disabled", sehCode);
WriteEventLog(buf, EVENTLOG_ERROR_TYPE);
this->_logger.LogError("ANSALPR_OCR::LoadEngine",
"Step 4: Colour classifier crashed. Colour detection disabled.", __FILE__, __LINE__);
if (_lpColourDetector) _lpColourDetector.reset();
}
else if (!colourSuccess) {
this->_logger.LogError("ANSALPR_OCR::LoadEngine",
"Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__);
if (_lpColourDetector) _lpColourDetector.reset();
}
}
}
valid = true;
_isInitialized = valid;
WriteEventLog(("ANSALPR_OCR::LoadEngine: Step 5 - Engine load complete. Valid = " + std::to_string(valid)).c_str());
this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 5: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__);
return valid;
}
catch (std::exception& e) {
WriteEventLog(("ANSALPR_OCR::LoadEngine: C++ exception: " + std::string(e.what())).c_str(), EVENTLOG_ERROR_TYPE);
this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", std::string("C++ exception: ") + e.what(), __FILE__, __LINE__);
_isInitialized = false;
return false;
}
catch (...) {
WriteEventLog("ANSALPR_OCR::LoadEngine: Unknown exception", EVENTLOG_ERROR_TYPE);
this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", "Unknown exception", __FILE__, __LINE__);
_isInitialized = false;
return false;
}
}
// ── Colour detection (same pattern as ANSALPR_OD) ────────────────────
std::string ANSALPR_OCR::DetectLPColourDetector(const cv::Mat& lprROI, const std::string& cameraId) {
if (_lpColourModelConfig.detectionScoreThreshold <= 0.0f) return {};
if (!_lpColourDetector) return {};
if (lprROI.empty()) return {};
try {
std::vector<Object> colourOutputs = _lpColourDetector->RunInference(lprROI, cameraId);
if (colourOutputs.empty()) return {};
const auto& bestDetection = *std::max_element(
colourOutputs.begin(), colourOutputs.end(),
[](const Object& a, const Object& b) { return a.confidence < b.confidence; }
);
return bestDetection.className;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSALPR_OCR::DetectLPColourDetector", e.what(), __FILE__, __LINE__);
return {};
}
}
std::string ANSALPR_OCR::DetectLPColourCached(const cv::Mat& lprROI, const std::string& cameraId, const std::string& plateText) {
if (plateText.empty()) {
return DetectLPColourDetector(lprROI, cameraId);
}
// Check cache first
{
std::lock_guard<std::mutex> cacheLock(_colourCacheMutex);
auto it = _colourCache.find(plateText);
if (it != _colourCache.end()) {
it->second.hitCount++;
return it->second.colour;
}
}
// Cache miss — run classifier
std::string colour = DetectLPColourDetector(lprROI, cameraId);
if (!colour.empty()) {
std::lock_guard<std::mutex> cacheLock(_colourCacheMutex);
if (_colourCache.size() >= COLOUR_CACHE_MAX_SIZE) {
_colourCache.clear();
}
_colourCache[plateText] = { colour, 0 };
}
return colour;
}
// ── OCR on a single plate ROI ────────────────────────────────────────
// Returns the plate text via the out-parameter and populates alprExtraInfo
// with the structured ALPR JSON (zone parts) when ALPR mode is active.
std::string ANSALPR_OCR::RunOCROnPlate(const cv::Mat& plateROI, const std::string& cameraId) {
if (!_ocrEngine || plateROI.empty()) return "";
if (plateROI.cols < 10 || plateROI.rows < 10) return "";
try {
// Run the full ANSONNXOCR pipeline on the cropped plate image
std::vector<OCRObject> ocrResults = _ocrEngine->RunInference(plateROI, cameraId);
if (ocrResults.empty()) return "";
// If ALPR mode is active and we have plate formats, use the
// structured ALPR post-processing to get correct zone ordering
// (e.g. "品川 302 ま 93-15" instead of "品川30293-15ま")
const auto& alprFormats = _ocrEngine->GetALPRFormats();
if (_ocrEngine->GetOCRMode() == OCRMode::OCR_ALPR && !alprFormats.empty()) {
auto alprResults = ANSOCRUtility::ALPRPostProcessing(
ocrResults, alprFormats,
plateROI.cols, plateROI.rows,
_ocrEngine.get(), plateROI);
if (!alprResults.empty()) {
return alprResults[0].fullPlateText;
}
}
// Fallback: simple concatenation sorted by Y then X
std::sort(ocrResults.begin(), ocrResults.end(),
[](const OCRObject& a, const OCRObject& b) {
int rowThreshold = std::min(a.box.height, b.box.height) / 2;
if (std::abs(a.box.y - b.box.y) > rowThreshold) {
return a.box.y < b.box.y;
}
return a.box.x < b.box.x;
}
);
std::string fullText;
for (const auto& obj : ocrResults) {
if (!obj.className.empty()) {
fullText += obj.className;
}
}
return fullText;
}
catch (const std::exception& e) {
this->_logger.LogError("ANSALPR_OCR::RunOCROnPlate", e.what(), __FILE__, __LINE__);
return "";
}
}
// ── Main inference pipeline ──────────────────────────────────────────
std::vector<Object> ANSALPR_OCR::RunInference(const cv::Mat& input, const std::string& cameraId) {
if (!_licenseValid) {
this->_logger.LogError("ANSALPR_OCR::RunInference", "Invalid license", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
this->_logger.LogError("ANSALPR_OCR::RunInference", "Model is not initialized", __FILE__, __LINE__);
return {};
}
if (input.empty() || input.cols < 5 || input.rows < 5) {
this->_logger.LogError("ANSALPR_OCR::RunInference", "Input image is empty or too small", __FILE__, __LINE__);
return {};
}
if (!_lpDetector) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_lpDetector is null", __FILE__, __LINE__);
return {};
}
if (!_ocrEngine) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_ocrEngine is null", __FILE__, __LINE__);
return {};
}
try {
// Convert grayscale to BGR if necessary
cv::Mat localFrame;
if (input.channels() == 1) {
cv::cvtColor(input, localFrame, cv::COLOR_GRAY2BGR);
}
const cv::Mat& frame = (input.channels() == 1) ? localFrame : input;
const int frameWidth = frame.cols;
const int frameHeight = frame.rows;
// Step 1: Detect license plates
std::vector<Object> lprOutput = _lpDetector->RunInference(frame, cameraId);
if (lprOutput.empty()) {
return {};
}
std::vector<Object> output;
output.reserve(lprOutput.size());
for (auto& lprObject : lprOutput) {
const cv::Rect& box = lprObject.box;
// Calculate safe cropped region
const int x1 = std::max(0, box.x);
const int y1 = std::max(0, box.y);
const int width = std::min(frameWidth - x1, box.width);
const int height = std::min(frameHeight - y1, box.height);
if (width <= 0 || height <= 0) continue;
cv::Rect lprPos(x1, y1, width, height);
cv::Mat plateROI = frame(lprPos);
// Step 2: Run OCR on the detected plate
std::string ocrText = RunOCROnPlate(plateROI, cameraId);
if (ocrText.empty()) continue;
lprObject.cameraId = cameraId;
// Use ALPRChecker for text stabilization if enabled
if (_enableALPRChecker) {
lprObject.className = alprChecker.checkPlateByTrackId(cameraId, ocrText, lprObject.trackId);
} else {
lprObject.className = ocrText;
}
if (lprObject.className.empty()) continue;
// Step 3: Colour detection (optional)
std::string colour = DetectLPColourCached(plateROI, cameraId, lprObject.className);
if (!colour.empty()) {
lprObject.extraInfo = "color:" + colour;
}
output.push_back(std::move(lprObject));
}
return output;
}
catch (const cv::Exception& e) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", std::string("OpenCV Exception: ") + e.what(), __FILE__, __LINE__);
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", e.what(), __FILE__, __LINE__);
}
catch (...) {
this->_logger.LogFatal("ANSALPR_OCR::RunInference", "Unknown exception occurred", __FILE__, __LINE__);
}
return {};
}
// ── Inference wrappers ───────────────────────────────────────────────
bool ANSALPR_OCR::Inference(const cv::Mat& input, std::string& lprResult) {
if (input.empty()) return false;
if (input.cols < 5 || input.rows < 5) return false;
return Inference(input, lprResult, "CustomCam");
}
bool ANSALPR_OCR::Inference(const cv::Mat& input, std::string& lprResult, const std::string& cameraId) {
if (input.empty()) return false;
if (input.cols < 5 || input.rows < 5) return false;
try {
std::vector<Object> results = RunInference(input, cameraId);
lprResult = VectorDetectionToJsonString(results);
return !results.empty();
}
catch (...) {
return false;
}
}
bool ANSALPR_OCR::Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult) {
return Inference(input, Bbox, lprResult, "CustomCam");
}
bool ANSALPR_OCR::Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult, const std::string& cameraId) {
if (input.empty()) return false;
if (input.cols < 5 || input.rows < 5) return false;
try {
if (Bbox.empty()) {
return Inference(input, lprResult, cameraId);
}
// For cropped images, run OCR on each bounding box
std::vector<Object> allResults;
cv::Mat frame;
if (input.channels() == 1) {
cv::cvtColor(input, frame, cv::COLOR_GRAY2BGR);
} else {
frame = input;
}
for (const auto& bbox : Bbox) {
int x1 = std::max(0, bbox.x);
int y1 = std::max(0, bbox.y);
int w = std::min(frame.cols - x1, bbox.width);
int h = std::min(frame.rows - y1, bbox.height);
if (w < 5 || h < 5) continue;
cv::Rect safeRect(x1, y1, w, h);
cv::Mat cropped = frame(safeRect);
std::vector<Object> results = RunInference(cropped, cameraId);
// Adjust bounding boxes back to full image coordinates
for (auto& obj : results) {
obj.box.x += x1;
obj.box.y += y1;
allResults.push_back(std::move(obj));
}
}
lprResult = VectorDetectionToJsonString(allResults);
return !allResults.empty();
}
catch (...) {
return false;
}
}
void ANSALPR_OCR::SetCountry(Country country) {
_country = country;
if (_ocrEngine) {
_ocrEngine->SetCountry(country);
}
}
bool ANSALPR_OCR::Destroy() {
try {
if (_lpDetector) {
_lpDetector->Destroy();
_lpDetector.reset();
}
if (_lpColourDetector) {
_lpColourDetector->Destroy();
_lpColourDetector.reset();
}
if (_ocrEngine) {
_ocrEngine->Destroy();
_ocrEngine.reset();
}
_isInitialized = false;
return true;
}
catch (std::exception& e) {
this->_logger.LogFatal("ANSALPR_OCR::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
} // namespace ANSCENTER

91
modules/ANSLPR/ANSLPR_OCR.h Normal file
View File

@@ -0,0 +1,91 @@
#ifndef ANSLPROCR_H
#define ANSLPROCR_H
#pragma once
#include "ANSLPR.h"
#include <list>
#include <map>
#include <string>
#include <mutex>
#include <utility>
#include <vector>
// Forward-declare ANSONNXOCR to avoid pulling in the full ANSOCR header chain
namespace ANSCENTER { class ANSONNXOCR; struct OCRModelConfig; }
namespace ANSCENTER
{
/// ANSALPR_OCR — License plate recognition using ONNX YOLO for LP detection
/// and ANSONNXOCR (PaddleOCR v5) for text recognition.
///
/// Pipeline:
/// 1. Detect license plates using _lpDetector (ANSONNXYOLO)
/// 2. For each detected plate, run OCR using _ocrEngine (ANSONNXOCR)
/// 3. Optionally classify plate colour using _lpColourDetector (ANSONNXYOLO)
///
/// Supports multiple countries via the Country enum and ALPR post-processing
/// from ANSOCR's ANSOCRBase infrastructure.
class ANSLPR_API ANSALPR_OCR : public ANSALPR {
private:
ANSCENTER::EngineType engineType;
// --- Detectors ---
std::unique_ptr<ANSCENTER::ANSODBase> _lpDetector = nullptr; // License plate detector
std::unique_ptr<ANSCENTER::ANSODBase> _lpColourDetector = nullptr; // License plate colour classifier
std::unique_ptr<ANSCENTER::ANSONNXOCR> _ocrEngine = nullptr; // OCR text recognizer
// --- Model configs ---
ANSCENTER::ModelConfig _lpdmodelConfig;
ANSCENTER::ModelConfig _lpColourModelConfig;
std::string _lpdLabels;
std::string _lpColourLabels;
cv::Mat _frameBuffer; // Reusable buffer for color conversion
std::vector<std::string> _lprModelClass;
ALPRChecker alprChecker;
// --- Original model zip path (reused for ANSONNXOCR initialization) ---
std::string _modelZipFilePath;
// --- Colour detection helpers ---
[[nodiscard]] std::string DetectLPColourDetector(const cv::Mat& lprROI, const std::string& cameraId);
[[nodiscard]] std::string DetectLPColourCached(const cv::Mat& lprROI, const std::string& cameraId, const std::string& plateText);
// LPC colour cache
struct ColourCacheEntry {
std::string colour;
int hitCount = 0;
};
std::mutex _colourCacheMutex;
std::unordered_map<std::string, ColourCacheEntry> _colourCache;
static constexpr size_t COLOUR_CACHE_MAX_SIZE = 200;
// --- OCR helper ---
[[nodiscard]] std::string RunOCROnPlate(const cv::Mat& plateROI, const std::string& cameraId);
public:
ANSALPR_OCR();
~ANSALPR_OCR();
[[nodiscard]] bool Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword, double detectorThreshold, double ocrThreshold, double colourThreshold) override;
[[nodiscard]] bool LoadEngine() override;
[[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult) override;
[[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult, const std::string& cameraId) override;
[[nodiscard]] bool Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult) override;
[[nodiscard]] bool Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult, const std::string& cameraId) override;
[[nodiscard]] std::vector<Object> RunInference(const cv::Mat& input, const std::string& cameraId) override;
[[nodiscard]] bool Destroy() override;
/// Propagate country to inner OCR engine so ALPR post-processing
/// uses the correct plate formats and character corrections.
void SetCountry(Country country) override;
/// Propagate debug flag to all sub-detectors
void ActivateDebugger(bool debugFlag) override {
_debugFlag = debugFlag;
if (_lpDetector) _lpDetector->ActivateDebugger(debugFlag);
if (_lpColourDetector) _lpColourDetector->ActivateDebugger(debugFlag);
}
};
}
#endif

267
modules/ANSLPR/ANSLPR_OD.cpp
View File

@@ -272,6 +272,82 @@ static bool LoadLpcModel_SEH(const LoadLpcParams& p, DWORD* outCode) {
return false;
}
}
// ---------------------------------------------------------------------------
// Generic SEH wrapper for loading an ANSONNXYOLO model (used by the CPU /
// AMD / Intel fallback path where TensorRT is unavailable).
//
// Why SEH is required here
// ------------------------
// DirectML / OpenVINO / CUDA ORT session creation can crash with an
// asynchronous hardware fault (STATUS_ACCESS_VIOLATION 0xC0000005) when
// the underlying provider driver is in a bad state. C++ `try/catch` does
// NOT catch SEH exceptions on MSVC unless the translator is explicitly
// installed. Without this SEH wrapper the AV propagates up through
// ANSALPR_OD::LoadEngine into LoadANSALPREngineHandle, which logs
// "SEH exception 0xC0000005 caught during engine load" and returns 0 —
// the user sees a generic error with no way to tell which detector
// (LPD / OCR / LPC) failed.
//
// Wrapping each detector creation lets us:
// 1. Isolate the failing detector without taking down the whole load.
// 2. Log a precise error message indicating which model crashed.
// 3. Let the caller zero out the unique_ptr so Destroy() won't run a
// half-initialised engine during cleanup.
// ---------------------------------------------------------------------------
struct LoadOnnxParams {
const std::string* licenseKey;
ANSCENTER::ModelConfig* config;
const std::string* modelFolder;
const char* modelName;
const char* classFile;
std::string* labels;
std::unique_ptr<ANSCENTER::ANSODBase>* detector;
bool enableTracker;
bool disableStabilization;
};
static bool LoadOnnxModel_Impl(const LoadOnnxParams& p) {
try {
auto onnxyolo = std::make_unique<ANSCENTER::ANSONNXYOLO>();
bool ok = onnxyolo->LoadModelFromFolder(
*p.licenseKey, *p.config, p.modelName, p.classFile,
*p.modelFolder, *p.labels);
if (!ok) {
return false;
}
if (p.enableTracker) {
onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, true);
} else {
onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, false);
}
if (p.disableStabilization) {
onnxyolo->SetStabilization(false);
}
*p.detector = std::move(onnxyolo); // upcast ANSONNXYOLO -> ANSODBase
return true;
}
catch (...) {
p.detector->reset();
return false;
}
}
static bool LoadOnnxModel_SEH(const LoadOnnxParams& p, DWORD* outCode) {
// IMPORTANT: a function containing __try/__except must not run C++
// destructors in the handler body — the CRT's SEH unwind can collide
// with C++ unwind and call std::terminate. We therefore defer any
// cleanup (unique_ptr::reset) to the caller, which runs outside the
// SEH context. This mirrors the LoadLpcModel_SEH pattern above.
*outCode = 0;
__try {
return LoadOnnxModel_Impl(p);
}
__except (EXCEPTION_EXECUTE_HANDLER) {
*outCode = GetExceptionCode();
return false;
}
}
//#define FNS_DEBUG
namespace ANSCENTER {
@@ -285,6 +361,12 @@ namespace ANSCENTER {
ANSALPR_OD::ANSALPR_OD() {
valid = false;
// Default to safest engine (CPU). LoadEngine() overrides this after
// CheckHardwareInformation() runs. We must not leave engineType
// uninitialised because vendor predicates (isNvidiaEngine() etc.)
// gate NV12/CUDA paths and could otherwise activate the CUDA runtime
// on AMD/Intel hardware.
engineType = ANSCENTER::EngineType::CPU;
};
ANSALPR_OD::~ANSALPR_OD() {
try {
@@ -485,8 +567,16 @@ namespace ANSCENTER {
WriteEventLog("ANSALPR_OD::LoadEngine: Step 2 - Checking hardware information");
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 2: Checking hardware information", __FILE__, __LINE__);
engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation();//
WriteEventLog(("ANSALPR_OD::LoadEngine: Step 2 complete - Engine type = " + std::to_string(static_cast<int>(engineType))).c_str());
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 2 complete: Engine type = " + std::to_string(static_cast<int>(engineType)), __FILE__, __LINE__);
const char* vendorTag =
isNvidiaEngine() ? "NVIDIA_GPU (TensorRT + NV12/CUDA fast path)" :
isAmdEngine() ? "AMD_GPU (DirectML via ONNX Runtime, NV12/CUDA DISABLED)" :
isIntelEngine() ? "OPENVINO_GPU (OpenVINO via ONNX Runtime, NV12/CUDA DISABLED)" :
"CPU (ONNX Runtime, NV12/CUDA DISABLED)";
WriteEventLog(("ANSALPR_OD::LoadEngine: Step 2 complete - Engine type = " +
std::to_string(static_cast<int>(engineType)) + " [" + vendorTag + "]").c_str());
this->_logger.LogInfo("ANSALPR_OD::LoadEngine",
"Step 2 complete: Engine type = " + std::to_string(static_cast<int>(engineType)) +
" [" + vendorTag + "]", __FILE__, __LINE__);
valid = false;
if (_lpDetector) _lpDetector.reset();
@@ -788,54 +878,132 @@ namespace ANSCENTER {
}
}
}
// ONNX Runtime fallback path (CPU or when TensorRT fails)
// ONNX Runtime fallback path (CPU / AMD / Intel — or NVIDIA when
// TensorRT build failed). Each detector is loaded through a
// dedicated SEH wrapper (LoadOnnxModel_SEH) so that an
// AV / STATUS_ACCESS_VIOLATION raised deep inside the ONNX
// Runtime session creator (e.g. from a misbehaving DirectML
// / OpenVINO / CUDA provider driver) does not tear down the
// whole LoadEngine call. The wrapper logs the exact detector
// that failed and zeros out the corresponding unique_ptr.
if (!valid) {
if (FileExist(lprModel) && (FileExist(ocrModel)))
{
bool lpSuccess = false, ocrSuccess = false;
// ── Step 6: LPD ─────────────────────────────────────
WriteEventLog("ANSALPR_OD::LoadEngine: Step 6 - Loading LP detector with ONNX Runtime");
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 6: Loading LP detector with ONNX Runtime", __FILE__, __LINE__);
_lpdmodelConfig.detectionType = DetectionType::DETECTION;
_lpdmodelConfig.modelType = ModelType::ONNXYOLO;
_lpdmodelConfig.modelType = ModelType::ONNXYOLO;
std::string _lprClasses;
_lpDetector = std::make_unique<ANSCENTER::ANSONNXYOLO>();// Yolo
bool lpSuccess = _lpDetector->LoadModelFromFolder(_licenseKey, _lpdmodelConfig, "lpd", "lpd.names", _modelFolder, _lprClasses);
if (!lpSuccess) {
this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__);
_lpDetector.reset();
}
else {
// Enable tracker on LP detector for stable bounding box tracking,
// but disable stabilization (no ghost plates — ALPRChecker handles text stabilization)
_lpDetector->SetTracker(TrackerType::BYTETRACK, true);
_lpDetector->SetStabilization(false);
{
LoadOnnxParams p{};
p.licenseKey = &_licenseKey;
p.config = &_lpdmodelConfig;
p.modelFolder = &_modelFolder;
p.modelName = "lpd";
p.classFile = "lpd.names";
p.labels = &_lprClasses;
p.detector = &_lpDetector;
p.enableTracker = true;
p.disableStabilization = true;
DWORD sehCode = 0;
lpSuccess = LoadOnnxModel_SEH(p, &sehCode);
if (sehCode != 0) {
char buf[256];
snprintf(buf, sizeof(buf),
"ANSALPR_OD::LoadEngine: Step 6 LPD SEH exception 0x%08X — LP detector disabled", sehCode);
WriteEventLog(buf, EVENTLOG_ERROR_TYPE);
this->_logger.LogFatal("ANSALPR_OD::LoadEngine",
"Step 6: LP detector crashed (SEH 0x" + std::to_string(sehCode) + "). LP detector disabled.",
__FILE__, __LINE__);
lpSuccess = false;
// Drop any half-initialised state outside SEH context.
if (_lpDetector) _lpDetector.reset();
}
else if (!lpSuccess) {
this->_logger.LogError("ANSALPR_OD::LoadEngine",
"Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__);
if (_lpDetector) _lpDetector.reset();
}
}
// ── Step 7: OCR ─────────────────────────────────────
WriteEventLog("ANSALPR_OD::LoadEngine: Step 7 - Loading OCR detector with ONNX Runtime");
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 7: Loading OCR detector with ONNX Runtime", __FILE__, __LINE__);
_ocrModelConfig.detectionType = DetectionType::DETECTION;
_ocrModelConfig.modelType = ModelType::ONNXYOLO;
_ocrDetector = std::make_unique<ANSCENTER::ANSONNXYOLO>();// Yolo
bool ocrSuccess = _ocrDetector->LoadModelFromFolder(_licenseKey, _ocrModelConfig, "ocr", "ocr.names", _modelFolder, _ocrLabels);
if (!ocrSuccess) {
this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load OCR detector (ONNX Runtime).", __FILE__, __LINE__);
_ocrDetector.reset();
}
else {
_ocrDetector->SetTracker(TrackerType::BYTETRACK, false);
_ocrModelConfig.modelType = ModelType::ONNXYOLO;
{
LoadOnnxParams p{};
p.licenseKey = &_licenseKey;
p.config = &_ocrModelConfig;
p.modelFolder = &_modelFolder;
p.modelName = "ocr";
p.classFile = "ocr.names";
p.labels = &_ocrLabels;
p.detector = &_ocrDetector;
p.enableTracker = false;
p.disableStabilization = false;
DWORD sehCode = 0;
ocrSuccess = LoadOnnxModel_SEH(p, &sehCode);
if (sehCode != 0) {
char buf[256];
snprintf(buf, sizeof(buf),
"ANSALPR_OD::LoadEngine: Step 7 OCR SEH exception 0x%08X — OCR detector disabled", sehCode);
WriteEventLog(buf, EVENTLOG_ERROR_TYPE);
this->_logger.LogFatal("ANSALPR_OD::LoadEngine",
"Step 7: OCR detector crashed (SEH 0x" + std::to_string(sehCode) + "). OCR detector disabled.",
__FILE__, __LINE__);
ocrSuccess = false;
// Drop any half-initialised state outside SEH context.
if (_ocrDetector) _ocrDetector.reset();
}
else if (!ocrSuccess) {
this->_logger.LogError("ANSALPR_OD::LoadEngine",
"Failed to load OCR detector (ONNX Runtime).", __FILE__, __LINE__);
if (_ocrDetector) _ocrDetector.reset();
}
}
// Check if we need to load the color model
// ── Step 8: LPC (optional) ──────────────────────────
if (FileExist(colorModel) && (_lpColourModelConfig.detectionScoreThreshold > 0)) {
WriteEventLog("ANSALPR_OD::LoadEngine: Step 8 - Loading colour classifier with ONNX Runtime");
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 8: Loading colour classifier with ONNX Runtime", __FILE__, __LINE__);
_lpColourModelConfig.detectionType = DetectionType::CLASSIFICATION;
_lpColourModelConfig.modelType = ModelType::ONNXYOLO;
_lpColourDetector = std::make_unique<ANSCENTER::ANSONNXYOLO>();// Classification with ONNX
bool colourSuccess = _lpColourDetector->LoadModelFromFolder(_licenseKey, _lpColourModelConfig, "lpc", "lpc.names", _modelFolder, _lpColourLabels);
if (!colourSuccess) {
this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__);
_lpColourDetector.reset();
}
else {
_lpColourDetector->SetTracker(TrackerType::BYTETRACK, false);
_lpColourModelConfig.modelType = ModelType::ONNXYOLO;
{
LoadOnnxParams p{};
p.licenseKey = &_licenseKey;
p.config = &_lpColourModelConfig;
p.modelFolder = &_modelFolder;
p.modelName = "lpc";
p.classFile = "lpc.names";
p.labels = &_lpColourLabels;
p.detector = &_lpColourDetector;
p.enableTracker = false;
p.disableStabilization = false;
DWORD sehCode = 0;
bool colourSuccess = LoadOnnxModel_SEH(p, &sehCode);
if (sehCode != 0) {
char buf[256];
snprintf(buf, sizeof(buf),
"ANSALPR_OD::LoadEngine: Step 8 LPC SEH exception 0x%08X — colour detection disabled", sehCode);
WriteEventLog(buf, EVENTLOG_ERROR_TYPE);
this->_logger.LogError("ANSALPR_OD::LoadEngine",
"Step 8: Colour classifier crashed (SEH 0x" + std::to_string(sehCode) + "). Colour detection disabled.",
__FILE__, __LINE__);
// Drop any half-initialised state outside SEH context.
if (_lpColourDetector) _lpColourDetector.reset();
}
else if (!colourSuccess) {
this->_logger.LogError("ANSALPR_OD::LoadEngine",
"Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__);
if (_lpColourDetector) _lpColourDetector.reset();
}
}
}
@@ -851,8 +1019,8 @@ namespace ANSCENTER {
}
}
_isInitialized = valid;
WriteEventLog(("ANSALPR_OD::LoadEngine: Step 8 - Engine load complete. Valid = " + std::to_string(valid)).c_str());
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 8: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__);
WriteEventLog(("ANSALPR_OD::LoadEngine: Step 9 - Engine load complete. Valid = " + std::to_string(valid)).c_str());
this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 9: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__);
return valid;
}
@@ -1467,8 +1635,11 @@ namespace ANSCENTER {
constexpr int padding = 10;
// --- Compute display→full-res scale (once per frame, cheap) ---
// NV12 GPU fast path is NVIDIA-only — cv::cuda::Stream/GpuMat
// touch the CUDA runtime even when the helper would early-return,
// which destabilises AMD/Intel hardware. Gate strictly on NVIDIA.
float scaleX = 1.f, scaleY = 1.f;
{
if (isNvidiaEngine()) {
auto* gpuData = tl_currentGpuFrame();
if (gpuData && gpuData->width > frame.cols && gpuData->height > frame.rows) {
scaleX = static_cast<float>(gpuData->width) / frame.cols;
@@ -1484,8 +1655,9 @@ namespace ANSCENTER {
cv::Mat lprImage;
// Try GPU NV12 crop (NVIDIA decode: NV12 still in GPU VRAM)
if (scaleX > 1.f) {
// Try GPU NV12 crop (NVIDIA decode: NV12 still in GPU VRAM).
// Skipped on AMD/Intel/CPU — see isNvidiaEngine() guard above.
if (isNvidiaEngine() && scaleX > 1.f) {
auto cropResult = _nv12Helper.tryNV12CropToBGR(
frame, 0, box, padding, scaleX, scaleY,
this->_logger, "LPR");
@@ -1635,8 +1807,11 @@ namespace ANSCENTER {
constexpr int padding = 10;
// --- Compute display→full-res scale (once per frame, cheap) ---
// NVIDIA-only NV12 fast path — see isNvidiaEngine() discussion
// above. cv::cuda::* types touch CUDA even inside the "guarded"
// helper, so we must not even read tl_currentGpuFrame() on AMD.
float scaleX2 = 1.f, scaleY2 = 1.f;
{
if (isNvidiaEngine()) {
auto* gpuData = tl_currentGpuFrame();
if (gpuData && gpuData->width > frame.cols && gpuData->height > frame.rows) {
scaleX2 = static_cast<float>(gpuData->width) / frame.cols;
@@ -1694,9 +1869,11 @@ namespace ANSCENTER {
lprObject.cameraId = cameraId;
lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows);
// Crop from full-res NV12 on GPU if available, otherwise display-res
// Crop from full-res NV12 on GPU if available, otherwise display-res.
// NV12 helper is NVIDIA-only — isNvidiaEngine() gate keeps
// CUDA runtime inactive on AMD/Intel/CPU hardware.
cv::Mat lprImage;
if (scaleX2 > 1.f) {
if (isNvidiaEngine() && scaleX2 > 1.f) {
auto cropResult = _nv12Helper.tryNV12CropToBGR(
frame, 0, lprObject.box, 0, scaleX2, scaleY2,
this->_logger, "LPR");
@@ -1760,10 +1937,12 @@ namespace ANSCENTER {
lprObject.cameraId = cameraId;
lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows);
// Crop from full-res NV12 on GPU if available, otherwise display-res
// Crop from full-res NV12 on GPU if available, otherwise display-res.
// NV12 helper is NVIDIA-only — isNvidiaEngine() gate keeps
// CUDA runtime inactive on AMD/Intel/CPU hardware.
cv::Rect lprPos(x1, y1, width, height);
cv::Mat lprImage;
if (scaleX2 > 1.f) {
if (isNvidiaEngine() && scaleX2 > 1.f) {
auto cropResult = _nv12Helper.tryNV12CropToBGR(
frame, 0, lprPos, 0, scaleX2, scaleY2,
this->_logger, "LPR");

24
modules/ANSLPR/ANSLPR_OD.h
View File

@@ -17,6 +17,30 @@ namespace ANSCENTER
class ANSLPR_API ANSALPR_OD :public ANSALPR {
private:
ANSCENTER::EngineType engineType;
// --------------------------------------------------------------
// Vendor predicates — use these to gate hardware-specific paths
// (NV12 GPU crop, CUDA helpers, DirectML quirks, OpenVINO tricks).
//
// Certain helpers like _nv12Helper.tryNV12CropToBGR() call into
// CUDA runtime (cv::cuda::Stream/GpuMat/cudaStream_t) unconditionally,
// which is unsafe on AMD/Intel hardware — cv::cuda::Stream's ctor
// touches the CUDA driver even when the helper would early-return.
// Always wrap those calls in isNvidiaEngine() before invoking.
// --------------------------------------------------------------
[[nodiscard]] bool isNvidiaEngine() const noexcept {
return engineType == ANSCENTER::EngineType::NVIDIA_GPU;
}
[[nodiscard]] bool isAmdEngine() const noexcept {
return engineType == ANSCENTER::EngineType::AMD_GPU;
}
[[nodiscard]] bool isIntelEngine() const noexcept {
return engineType == ANSCENTER::EngineType::OPENVINO_GPU;
}
[[nodiscard]] bool isCpuEngine() const noexcept {
return engineType == ANSCENTER::EngineType::CPU;
}
std::unique_ptr<ANSCENTER::ANSODBase>_lpDetector = nullptr; // License plate detector
std::unique_ptr<ANSCENTER::ANSODBase>_ocrDetector = nullptr; // OCR detector
std::unique_ptr<ANSCENTER::ANSODBase>_lpColourDetector = nullptr; // License plate colour classifier

328
modules/ANSLPR/ANSLPR_OV.h
View File

@@ -1,328 +0,0 @@
#ifndef ANSLPROV_H
#define ANSLPROV_H
#pragma once
#include "ANSLPR.h"
#include <list>
#include <map>
#include <string>
#include <utility>
#include <vector>
#include "openvino/openvino.hpp"
#include "openvino/runtime/intel_gpu/properties.hpp"
#include "utils/args_helper.hpp"
#include "utils/input_wrappers.hpp"
#include "utils/common.hpp"
#include "utils/ocv_common.hpp"
#include "utils/threads_common.hpp"
#include "utils/grid_mat.hpp"
#include "monitors/presenter.h"
namespace ANSCENTER
{
void tryPush(const std::weak_ptr<Worker>& worker, std::shared_ptr<Task>&& task);
struct BboxAndDescr {
enum class ObjectType {
NONE,
VEHICLE,
PLATE,
} objectType;
cv::Rect rect;
std::string descr;
};
class Detector {
public:
struct Result {
std::size_t label;
float confidence;
cv::Rect location;
};
static constexpr int maxProposalCount = 200;
static constexpr int objectSize = 7; // Output should have 7 as a last dimension"
Detector() = default;
Detector(ov::Core& core,
const std::string& deviceName,
const std::string& xmlPath,
const std::vector<float>& detectionTresholds,
const bool autoResize);
ov::InferRequest createInferRequest();
void setImage(ov::InferRequest& inferRequest, const cv::Mat& img);
std::list<Result> getResults(ov::InferRequest& inferRequest, cv::Size upscale, std::vector<std::string>& rawResults);
private:
bool m_autoResize;
std::vector<float> m_detectionTresholds;
std::string m_detectorInputName;
std::string m_detectorOutputName;
ov::CompiledModel m_compiled_model;
};
class VehicleAttributesClassifier {
public:
VehicleAttributesClassifier() = default;
VehicleAttributesClassifier(ov::Core& core,
const std::string& deviceName,
const std::string& xmlPath,
const bool autoResize);
ov::InferRequest createInferRequest();
void setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect vehicleRect);
std::pair<std::string, std::string> getResults(ov::InferRequest& inferRequest);
private:
bool m_autoResize;
std::string m_attributesInputName;
std::string m_outputNameForColor;
std::string m_outputNameForType;
ov::CompiledModel m_compiled_model;
};
class Lpr {
public:
Lpr() = default;
Lpr(ov::Core& core,
const std::string& deviceName,
const std::string& xmlPath,
const bool autoResize);
ov::InferRequest createInferRequest();
void setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect plateRect);
std::string getResults(ov::InferRequest& inferRequest);
private:
bool m_autoResize;
int m_maxSequenceSizePerPlate = 0;
std::string m_LprInputName;
std::string m_LprInputSeqName;
std::string m_LprOutputName;
ov::Layout m_modelLayout;
ov::CompiledModel m_compiled_model;
};
//Utilities
struct InferRequestsContainer {
InferRequestsContainer() = default;
InferRequestsContainer(const InferRequestsContainer&) = delete;
InferRequestsContainer& operator=(const InferRequestsContainer&) = delete;
void assign(const std::vector<ov::InferRequest>& inferRequests) {
actualInferRequests = inferRequests;
this->inferRequests.container.clear();
for (auto& ir : this->actualInferRequests) {
this->inferRequests.container.push_back(ir);
}
}
ConcurrentContainer<std::vector<std::reference_wrapper<ov::InferRequest>>> inferRequests;
std::vector<ov::InferRequest> actualInferRequests;
};
// stores all global data for tasks
struct Context {
Context(const std::vector<std::shared_ptr<InputChannel>>& inputChannels,
const Detector& detector,
const VehicleAttributesClassifier& vehicleAttributesClassifier,
const Lpr& lpr,
uint64_t lastFrameId,
uint64_t nireq,
bool isVideo,
std::size_t nclassifiersireq,
std::size_t nrecognizersireq) :
readersContext{ inputChannels, std::vector<int64_t>(inputChannels.size(), -1), std::vector<std::mutex>(inputChannels.size()) },
inferTasksContext{ detector },
detectionsProcessorsContext{ vehicleAttributesClassifier, lpr },
videoFramesContext{ std::vector<uint64_t>(inputChannels.size(), lastFrameId), std::vector<std::mutex>(inputChannels.size()) },
nireq{ nireq },
isVideo{ isVideo },
freeDetectionInfersCount{ 0 },
frameCounter{ 0 }
{
std::vector<ov::InferRequest> detectorInferRequests;
std::vector<ov::InferRequest> attributesInferRequests;
std::vector<ov::InferRequest> lprInferRequests;
detectorInferRequests.reserve(nireq);
attributesInferRequests.reserve(nclassifiersireq);
lprInferRequests.reserve(nrecognizersireq);
std::generate_n(std::back_inserter(detectorInferRequests), nireq, [&] {return inferTasksContext.detector.createInferRequest(); });
std::generate_n(std::back_inserter(attributesInferRequests), nclassifiersireq, [&] {return detectionsProcessorsContext.vehicleAttributesClassifier.createInferRequest(); });
std::generate_n(std::back_inserter(lprInferRequests), nrecognizersireq, [&] {return detectionsProcessorsContext.lpr.createInferRequest(); });
detectorsInfers.assign(detectorInferRequests);
attributesInfers.assign(attributesInferRequests);
platesInfers.assign(lprInferRequests);
}
struct {
std::vector<std::shared_ptr<InputChannel>> inputChannels;
std::vector<int64_t> lastCapturedFrameIds;
std::vector<std::mutex> lastCapturedFrameIdsMutexes;
std::weak_ptr<Worker> readersWorker;
} readersContext;
struct {
Detector detector;
std::weak_ptr<Worker> inferTasksWorker;
} inferTasksContext;
struct {
VehicleAttributesClassifier vehicleAttributesClassifier;
Lpr lpr;
std::weak_ptr<Worker> detectionsProcessorsWorker;
} detectionsProcessorsContext;
struct {
std::vector<uint64_t> lastframeIds;
std::vector<std::mutex> lastFrameIdsMutexes;
} videoFramesContext;
std::weak_ptr<Worker> resAggregatorsWorker;
std::mutex classifiersAggregatorPrintMutex;
uint64_t nireq;
bool isVideo;
std::atomic<std::vector<ov::InferRequest>::size_type> freeDetectionInfersCount;
std::atomic<uint32_t> frameCounter;
InferRequestsContainer detectorsInfers, attributesInfers, platesInfers;
PerformanceMetrics metrics;
std::list<BboxAndDescr> boxesAndDescrs;
};
// End of Context
class ReborningVideoFrame : public VideoFrame {
public:
ReborningVideoFrame(Context& context,
const unsigned sourceID,
const int64_t frameId,
const cv::Mat& frame = cv::Mat()) :
VideoFrame{ sourceID, frameId, frame },
context(context) {}
virtual ~ReborningVideoFrame();
Context& context;
};
// draws results on the frame
class ResAggregator : public Task {
public:
ResAggregator(const VideoFrame::Ptr& sharedVideoFrame,
std::list<BboxAndDescr>&& boxesAndDescrs) :
Task{ sharedVideoFrame, 4.0 },
boxesAndDescrs{ std::move(boxesAndDescrs) } {}
bool isReady() override {
return true;
}
void process() override;
private:
std::list<BboxAndDescr> boxesAndDescrs;
};
// waits for all classifiers and recognisers accumulating results
class ClassifiersAggregator {
public:
std::vector<std::string> rawDetections;
ConcurrentContainer<std::list<std::string>> rawAttributes;
ConcurrentContainer<std::list<std::string>> rawDecodedPlates;
explicit ClassifiersAggregator(const VideoFrame::Ptr& sharedVideoFrame) :
sharedVideoFrame{ sharedVideoFrame } {}
~ClassifiersAggregator() {
std::mutex& printMutex = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex;
printMutex.lock();
if (!rawDetections.empty()) {
slog::debug << "Frame #: " << sharedVideoFrame->frameId << slog::endl;
slog::debug << rawDetections;
// destructor assures that none uses the container
for (const std::string& rawAttribute : rawAttributes.container) {
slog::debug << rawAttribute << slog::endl;
}
for (const std::string& rawDecodedPlate : rawDecodedPlates.container) {
slog::debug << rawDecodedPlate << slog::endl;
}
}
printMutex.unlock();
tryPush(static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context.resAggregatorsWorker,
std::make_shared<ResAggregator>(sharedVideoFrame, std::move(boxesAndDescrs)));
}
void push(BboxAndDescr&& bboxAndDescr) {
boxesAndDescrs.lockedPushBack(std::move(bboxAndDescr));
}
const VideoFrame::Ptr sharedVideoFrame;
private:
ConcurrentContainer<std::list<BboxAndDescr>> boxesAndDescrs;
};
// extracts detections from blob InferRequests and runs classifiers and recognisers
class DetectionsProcessor : public Task {
public:
DetectionsProcessor(VideoFrame::Ptr sharedVideoFrame, ov::InferRequest* inferRequest) :
Task{ sharedVideoFrame, 1.0 },
inferRequest{ inferRequest },
requireGettingNumberOfDetections{ true }
{
}
DetectionsProcessor(VideoFrame::Ptr sharedVideoFrame,
std::shared_ptr<ClassifiersAggregator>&& classifiersAggregator,
std::list<cv::Rect>&& vehicleRects,
std::list<cv::Rect>&& plateRects) :
Task{ sharedVideoFrame, 1.0 },
classifiersAggregator{ std::move(classifiersAggregator) },
inferRequest{ nullptr },
vehicleRects{ std::move(vehicleRects) },
plateRects{ std::move(plateRects) },
requireGettingNumberOfDetections{ false }
{
}
bool isReady() override;
void process() override;
private:
std::shared_ptr<ClassifiersAggregator> classifiersAggregator; // when no one stores this object we will draw
ov::InferRequest* inferRequest;
std::list<cv::Rect> vehicleRects;
std::list<cv::Rect> plateRects;
std::vector<std::reference_wrapper<ov::InferRequest>> reservedAttributesRequests;
std::vector<std::reference_wrapper<ov::InferRequest>> reservedLprRequests;
bool requireGettingNumberOfDetections;
};
// runs detection
class InferTask : public Task {
public:
explicit InferTask(VideoFrame::Ptr sharedVideoFrame) :
Task{ sharedVideoFrame, 5.0 } {}
bool isReady() override;
void process() override;
};
class Reader : public Task {
public:
explicit Reader(VideoFrame::Ptr sharedVideoFrame) :
Task{ sharedVideoFrame, 2.0 } {}
bool isReady() override;
void process() override;
};
class ANSLPR_API ANSALPR_OV :public ANSALPR {
private:
std::string _vehicleLPModel; //model that detects both vehicle and license plate
std::string _vehicleAtModel; //model that detects vehicle attributes
std::string _lprModel; //model that recognise license plate
Detector* _detector = nullptr;
VehicleAttributesClassifier* _vehicleAttributesClassifier = nullptr;
Lpr* _lpr = nullptr;
[[nodiscard]] std::string VectorDetectionToJsonString(const std::vector<ALPRObject>& dets);
public:
ANSALPR_OV();
~ANSALPR_OV();
[[nodiscard]] bool Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
[[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult);
[[nodiscard]] bool Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult);
[[nodiscard]] bool Destroy() override;
};
}
#endif

776
modules/ANSLPR/ANSLPR_RT.cpp
View File

@@ -1,776 +0,0 @@
#include "ANSLPR_RT.h"
namespace ANSCENTER {
ANSALPR_RT::ANSALPR_RT() {
_licenseValid = false;
_globalViewId = 0;
_focusedOnLPRId = 0;
_platesTypesClassifierId = 0;
}
ANSALPR_RT::~ANSALPR_RT() {
CloseReferences();
}
bool ANSALPR_RT::Destroy() {
CloseReferences();
return true;
}
bool ANSALPR_RT::CloseReferences() {
try {
bool session_closed = CloseDetector(_globalViewId);
if (!session_closed) {
this->_logger->LogFatal("ANSALPR_RT::CloseReferences", "Cannot close global view model reference", __FILE__, __LINE__);
}
else {
_globalViewId = 0;
}
session_closed = CloseDetector(_focusedOnLPRId);
if (!session_closed) {
this->_logger->LogFatal("ANSALPR_RT::CloseReferences", "Cannot close focused on LRP model reference", __FILE__, __LINE__);
}
else {
_focusedOnLPRId = 0;
}
session_closed = ClosePlatesTypesClassifier(_platesTypesClassifierId);
if (!session_closed) {
this->_logger->LogFatal("ANSALPR_RT::CloseReferences", "Cannot close plate type classifer model reference", __FILE__, __LINE__);
}
else {
_platesTypesClassifierId = 0;
}
return true;
}
catch (std::exception& e) {
this->_logger->LogFatal("ANSALPR_RT::CloseReferences", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSALPR_RT::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) {
try {
_licenseKey = licenseKey;
_licenseValid = false;
CheckLicense();
if (!_licenseValid) {
this->_logger->LogError("ANSALPR::Initialize.", "License is not valid.", __FILE__, __LINE__);
return false;
}
// Extract model folder
// 0. Check if the modelZipFilePath exist?
if (!FileExist(modelZipFilePath)) {
this->_logger->LogFatal("ANSALPR::Initialize", "Model zip file is not exist", __FILE__, __LINE__);
}
// 1. Unzip model zip file to a special location with folder name as model file (and version)
std::string outputFolder;
std::vector<std::string> passwordArray;
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
passwordArray.push_back("AnsDemoModels20@!");
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
//this->_logger->LogInfo("ANSFDBase::Initialize. Model name", modelName);
size_t vectorSize = passwordArray.size();
for (size_t i = 0; i < vectorSize; i++) {
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
break; // Break the loop when the condition is met.
}
// 2. Check if the outputFolder exist
if (!FolderExist(_modelFolder)) {
this->_logger->LogError("ANSFDBase::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
return false; // That means the model file is not exist or the password is not correct
}
//3. Get License plate models
_globalModelFileName = CreateFilePath(_modelFolder, "anslpr_alpr_focused_on_lp.onnx");
if (!FileExist(_globalModelFileName)) {
this->_logger->LogError("ANSALPR::Initialize. Global view model does not exist", _globalModelFileName, __FILE__, __LINE__);
return false;
}
_focusedOnLPRModelFileName = CreateFilePath(_modelFolder, "anslpr_alpr_global_view.onnx");
if (!FileExist(_focusedOnLPRModelFileName)) {
this->_logger->LogError("ANSALPR::Initialize. Focused On LRP model does not exist", _focusedOnLPRModelFileName, __FILE__, __LINE__);
return false;
}
_platesTypesClassifierFileName = CreateFilePath(_modelFolder, "plates_types_7.onnx");
if (!FileExist(_platesTypesClassifierFileName)) {
this->_logger->LogError("ANSALPR::Initialize.LRP classifier model does not exist", _platesTypesClassifierFileName, __FILE__, __LINE__);
return false;
}
_platesTypesLabelsFileName = CreateFilePath(_modelFolder, "plates_types_7.txt");
if (!FileExist(_platesTypesLabelsFileName)) {
this->_logger->LogError("ANSALPR::Initialize.LRP classifier label model does not exist", _platesTypesLabelsFileName, __FILE__, __LINE__);
return false;
}
// Load models
size_t len = _globalModelFileName.size();
//step 1 : Initializes a new detector by loading its model file. In return, you get a unique id. The repo comes with two models namely anslpr_alpr_focused_on_lpand anslpr_alpr_global_view.
//So you have to call this function twice to initialize both models.
_globalViewId = InitYoloDetector(len, _globalModelFileName.c_str());
if (_globalViewId <= 0) {
this->_logger->LogError("ANSALPR::Initialize.", "Global view model cannot be loaded", __FILE__, __LINE__);
return false;
}
len = _focusedOnLPRModelFileName.size();
_focusedOnLPRId = InitYoloDetector(len, _focusedOnLPRModelFileName.c_str());
if (_focusedOnLPRId <= 0) {
this->_logger->LogError("ANSALPR::Initialize.", "Focused on LPR model cannot be loaded", __FILE__, __LINE__);
return false;
}
len = _platesTypesClassifierFileName.size();
_platesTypesClassifierId = InitPlatesClassifer(len, _platesTypesClassifierFileName.c_str(), _platesTypesLabelsFileName.size(), _platesTypesLabelsFileName.c_str());
if (_platesTypesClassifierId <= 0) {
this->_logger->LogError("ANSALPR::Initialize.", "Plate type classifier model cannot be loaded", __FILE__, __LINE__);
return false;
}
return true;
}
catch (std::exception& e) {
this->_logger->LogFatal("ANSALPR::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSALPR_RT::Inference(const cv::Mat& input, std::string& lprResult)
{
if (!_licenseValid) {
lprResult = "";
return false;
}
try {
const size_t lpn_len = 15;
char* lpn = new char[lpn_len + 1];
std::vector<ALPRObject> output;
output.clear();
cv::Rect bbox;
bool detected = TwoStageLPRPlatesTypeDetection(
input.cols,//width of image
input.rows,//height of image i.e. the specified dimensions of the image
input.channels(),// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
input.data,
input.step,// source image bytes buffer
_globalViewId,
_focusedOnLPRId,//id : unique interger to identify the detector to be used
_platesTypesClassifierId,//unique id to identify the platestype classifier
lpn_len,
lpn, bbox);
lprResult = lpn;
if (detected) {
ALPRObject result;
result.classId = 0;
result.className = lpn;
result.confidence = 1.0;
result.box = bbox;
output.push_back(result);
}
lprResult = VectorDetectionToJsonString(output);
return detected;
}
catch (std::exception& e) {
this->_logger->LogFatal("ANSALPR::CheckStatus", e.what(), __FILE__, __LINE__);
lprResult = "";
return false;
};
}
bool ANSALPR_RT::Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult) {
if (!_licenseValid) {
lprResult = "";
return false;
}
try {
const size_t lpn_len = 15;
char* lpn = new char[lpn_len + 1];
std::vector<ALPRObject> output;
output.clear();
bool detected = false;
if (Bbox.size() > 0) {
cv::Mat frame = input.clone();
for (std::vector<cv::Rect>::iterator it = Bbox.begin(); it != Bbox.end(); it++) {
int x1, y1, x2, y2;
x1 = (*it).x;
y1 = (*it).y;
x2 = (*it).x + (*it).width;
y2 = (*it).y + (*it).height;
// Get cropped objects
cv::Rect objectPos(cv::Point(x1, y1), cv::Point(x2, y2));
cv::Mat croppedObject = frame(objectPos);
cv::Rect bbox;
detected = TwoStageLPRPlatesTypeDetection(
croppedObject.cols,//width of image
croppedObject.rows,//height of image i.e. the specified dimensions of the image
croppedObject.channels(),// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
croppedObject.data,
croppedObject.step,// source image bytes buffer
_globalViewId,
_focusedOnLPRId,//id : unique interger to identify the detector to be used
_platesTypesClassifierId,//unique id to identify the platestype classifier
lpn_len,
lpn, bbox);
lprResult = lpn;
if (detected) {
ALPRObject result;
result.classId = 0;
result.className = lpn;
result.confidence = 1.0;
result.box.x = bbox.x + x1;
result.box.y = bbox.y + y1;
result.box.width = bbox.width;
result.box.height = bbox.height;
output.push_back(result);
}
}
lprResult = VectorDetectionToJsonString(output);
}
else {
cv::Rect bbox;
detected = TwoStageLPRPlatesTypeDetection(
input.cols,//width of image
input.rows,//height of image i.e. the specified dimensions of the image
input.channels(),// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
input.data,
input.step,// source image bytes buffer
_globalViewId,
_focusedOnLPRId,//id : unique interger to identify the detector to be used
_platesTypesClassifierId,//unique id to identify the platestype classifier
lpn_len,
lpn, bbox);
lprResult = lpn;
if (detected) {
ALPRObject result;
result.classId = 0;
result.className = lpn;
result.confidence = 1.0;
result.box = bbox;
output.push_back(result);
}
lprResult = VectorDetectionToJsonString(output);
}
return detected;
}
catch (std::exception& e) {
this->_logger->LogFatal("ANSALPR::CheckStatus", e.what(), __FILE__, __LINE__);
lprResult = "";
return false;
};
}
// Private:
std::string ANSALPR_RT::VectorDetectionToJsonString(const std::vector<ALPRObject>& dets) {
boost::property_tree::ptree root;
boost::property_tree::ptree detectedObjects;
for (int i = 0; i < dets.size(); i++) {
boost::property_tree::ptree detectedNode;
detectedNode.put("class_id", dets[i].classId);
detectedNode.put("class_name", dets[i].className);
detectedNode.put("prob", dets[i].confidence);
detectedNode.put("x", dets[i].box.x);
detectedNode.put("y", dets[i].box.y);
detectedNode.put("width", dets[i].box.width);
detectedNode.put("height", dets[i].box.height);
detectedNode.put("mask", "");//Todo: convert masks to mask with comma seperated dets[i].mask);
detectedNode.put("extra_info", "");
// we might add masks into this using comma seperated string
detectedObjects.push_back(std::make_pair("", detectedNode));
}
root.add_child("results", detectedObjects);
std::ostringstream stream;
boost::property_tree::write_json(stream, root, false);
std::string trackingResult = stream.str();
return trackingResult;
}
void ANSALPR_RT::CheckStatus(OrtStatus* status) {
try {
if (status != nullptr) {
const char* msg = g_ort->GetErrorMessage(status);
this->_logger->LogError("ANSALPR::CheckStatus", msg, __FILE__, __LINE__);
g_ort->ReleaseStatus(status);
exit(1);
}
}
catch (std::exception& e) {
this->_logger->LogFatal("ANSALPR::CheckStatus", e.what(), __FILE__, __LINE__);
}
}
std::list<Yolov5_alpr_onxx_detector*>::const_iterator ANSALPR_RT::GetDetector(unsigned int id, const std::list<Yolov5_alpr_onxx_detector*>& detectors,
const std::list<unsigned int>& detectors_ids)
{
assert(detectors_ids.size() == detectors.size());
std::list<Yolov5_alpr_onxx_detector*>::const_iterator it(detectors.begin());
std::list<unsigned int>::const_iterator it_id(detectors_ids.begin());
while (it != detectors.end() && it_id != detectors_ids.end()) {
if (*it_id == id) {
return it;
}
else {
it_id++;
it++;
}
}
return detectors.end();
}
std::list<Plates_types_classifier*>::const_iterator ANSALPR_RT::GetPlatesTypesClassifier(unsigned int id, const std::list<Plates_types_classifier*>& plates_types_classifiers,
const std::list<unsigned int>& plates_types_classifiers_ids) {
assert(plates_types_classifiers_ids.size() == plates_types_classifiers.size());
std::list<Plates_types_classifier*>::const_iterator it(plates_types_classifiers.begin());
std::list<unsigned int>::const_iterator it_id(plates_types_classifiers_ids.begin());
while (it != plates_types_classifiers.end() && it_id != plates_types_classifiers_ids.end()) {
if (*it_id == id) {
return it;
}
else {
it_id++;
it++;
}
}
return plates_types_classifiers.end();
}
unsigned int ANSALPR_RT::GetNewId(const std::list<unsigned int>& detectors_ids) {
if (detectors_ids.size()) {
auto result = std::minmax_element(detectors_ids.begin(), detectors_ids.end());
return *result.second + 1;
}
else return 1;
}
bool ANSALPR_RT::CloseDetector(unsigned int id, std::list<Ort::Env*>& _envs, std::list<Ort::SessionOptions*>& _lsessionOptions, std::list<Yolov5_alpr_onxx_detector*>& _detectors,
std::list<unsigned int>& _detectors_ids) {
assert(_detectors_ids.size() == _detectors.size()
&& _detectors_ids.size() == _envs.size()
&& _detectors_ids.size() == _lsessionOptions.size());
std::list<Yolov5_alpr_onxx_detector*>::iterator it(_detectors.begin());
std::list<unsigned int>::iterator it_id(_detectors_ids.begin());
std::list<Ort::SessionOptions*>::iterator it_sessionOptions(_lsessionOptions.begin());
std::list<Ort::Env*>::iterator it_envs(_envs.begin());
while (it != _detectors.end() && it_id != _detectors_ids.end()
&& it_envs != _envs.end() && it_sessionOptions != _lsessionOptions.end()
) {
if (*it_id == id) {
if (*it != nullptr) delete* it;
if (*it_sessionOptions != nullptr) delete* it_sessionOptions;
if (*it_envs != nullptr) delete* it_envs;
it_envs = _envs.erase(it_envs);
it_sessionOptions = _lsessionOptions.erase(it_sessionOptions);
it = _detectors.erase(it);
it_id = _detectors_ids.erase(it_id);
return true;
}
else {
it_sessionOptions++;
it_envs++;
it_id++;
it++;
}
}
return false;
}
bool ANSALPR_RT::CloseDetector(unsigned int id, std::list<Ort::Env*>& _envs, std::list<Ort::SessionOptions*>& _lsessionOptions, std::list<Plates_types_classifier*>& _detectors,
std::list<unsigned int>& _detectors_ids) {
assert(_detectors_ids.size() == _detectors.size()
&& _detectors_ids.size() == _envs.size()
&& _detectors_ids.size() == _lsessionOptions.size());
std::list<Plates_types_classifier*>::iterator it(_detectors.begin());
std::list<unsigned int>::iterator it_id(_detectors_ids.begin());
std::list<Ort::SessionOptions*>::iterator it_sessionOptions(_lsessionOptions.begin());
std::list<Ort::Env*>::iterator it_envs(_envs.begin());
while (it != _detectors.end() && it_id != _detectors_ids.end()
&& it_envs != _envs.end() && it_sessionOptions != _lsessionOptions.end()
) {
if (*it_id == id) {
if (*it != nullptr) delete* it;
if (*it_sessionOptions != nullptr) delete* it_sessionOptions;
if (*it_envs != nullptr) delete* it_envs;
it_envs = _envs.erase(it_envs);
it_sessionOptions = _lsessionOptions.erase(it_sessionOptions);
it = _detectors.erase(it);
it_id = _detectors_ids.erase(it_id);
return true;
}
else {
it_sessionOptions++;
it_envs++;
it_id++;
it++;
}
}
return false;
}
// Private interface
unsigned int ANSALPR_RT::InitYoloDetector(unsigned int len, const char* model_file)
{
assert(detectors_ids.size() == detectors.size());
const std::string model_filename(model_file, len);
if (!model_filename.size() || !std::filesystem::exists(model_filename)
|| !std::filesystem::is_regular_file(model_filename)
)
{
this->_logger->LogError("ANSALPR::InitYoloDetector. Model file is not regular file.", model_filename, __FILE__, __LINE__);
return 0;
}
//step 2 declare an onnx runtime environment
std::string instanceName{ "image-classification-inference" };
// https://github.com/microsoft/onnxruntime/blob/rel-1.6.0/include/onnxruntime/core/session/onnxruntime_c_api.h#L123
Ort::Env* penv = new Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str());
if (penv != nullptr) {
//step 3 declare options for the runtime environment
Ort::SessionOptions* psessionOptions = new Ort::SessionOptions();
if (psessionOptions != nullptr) {
psessionOptions->SetIntraOpNumThreads(1);
// Sets graph optimization level
// Available levels are
// ORT_DISABLE_ALL -> To disable all optimizations
// ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node
// removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations
// (Includes level 1 + more complex optimizations like node fusions)
// ORT_ENABLE_ALL -> To Enable All possible optimizations
psessionOptions->SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
#ifdef ANSLPR_USE_CUDA
// Optionally add more execution providers via session_options
// E.g. for CUDA include cuda_provider_factory.h and uncomment the following line:
// nullptr for Status* indicates success
OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(*psessionOptions, 0);
//or status =nullptr; //if you don t have CUDA
if (status == nullptr) {
#endif //ANSLPR_USE_CUDA
Yolov5_alpr_onxx_detector* onnx_net = nullptr;
#ifdef _WIN32
//step 4 declare an onnx session (ie model), by giving references to the runtime environment, session options and file path to the model
std::wstring widestr = std::wstring(model_filename.begin(), model_filename.end());
onnx_net = new Yolov5_alpr_onxx_detector(*penv, widestr.c_str(), *psessionOptions);
#else
onnx_net = new Yolov5_alpr_onxx_detector(*penv, model_filename.c_str(), *psessionOptions);
#endif
if (onnx_net != nullptr && penv != nullptr && psessionOptions != nullptr) {
std::unique_lock<std::mutex> lck(mtxMutex, std::defer_lock);
lck.lock();
detectors_envs.push_back(penv);
l_detectors_sessionOptions.push_back(psessionOptions);
detectors.push_back(onnx_net);
unsigned int id = GetNewId(detectors_ids);
detectors_ids.push_back(id);
lck.unlock();
return id;
}
else {
this->_logger->LogError("ANSALPR::InitYoloDetector. Error while creating onnxruntime session with file", model_filename.c_str(), __FILE__, __LINE__);
return 0;
}
#ifdef ANSLPR_USE_CUDA
}
else {
CheckStatus(status);
this->_logger->LogError("ANSALPR::InitYoloDetector.", "Cuda error", __FILE__, __LINE__);
return 0;
}
#endif //ANSLPR_USE_CUDA
}
else {
this->_logger->LogError("ANSALPR::InitYoloDetector.", "Error while creating SessionOptions", __FILE__, __LINE__);
return 0;
}
}
else {
this->_logger->LogError("ANSALPR::InitYoloDetector.", "Error while creating while creating session environment (Ort::Env)", __FILE__, __LINE__);
return 0;
}
}
unsigned int ANSALPR_RT::InitPlatesClassifer(unsigned int len_models_filename, const char* model_file, unsigned int len_labels_filename, const char* labels_file)
{
assert(plates_types_classifier_ids.size() == plates_types_classifiers.size());
const std::string model_filename(model_file, len_models_filename);
const std::string labels_filename(labels_file, len_labels_filename);
if (!model_filename.size() || !std::filesystem::exists(model_filename)
|| !std::filesystem::is_regular_file(model_filename)
|| !labels_filename.size() || !std::filesystem::exists(labels_filename)
|| !std::filesystem::is_regular_file(labels_filename)
)
{
this->_logger->LogDebug("ANSALPR::InitPlatesClassifer. Model file is not regular file.", model_filename, __FILE__, __LINE__);
return 0;
}
//step 2 declare an onnx runtime environment
std::string instanceName{ "image-classification-inference" };
// https://github.com/microsoft/onnxruntime/blob/rel-1.6.0/include/onnxruntime/core/session/onnxruntime_c_api.h#L123
Ort::Env* penv = new Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str());
if (penv != nullptr) {
//step 3 declare options for the runtime environment
Ort::SessionOptions* psessionOptions = new Ort::SessionOptions();
if (psessionOptions != nullptr) {
psessionOptions->SetIntraOpNumThreads(1);
// Sets graph optimization level
// Available levels are
// ORT_DISABLE_ALL -> To disable all optimizations
// ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node
// removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations
// (Includes level 1 + more complex optimizations like node fusions)
// ORT_ENABLE_ALL -> To Enable All possible optimizations
psessionOptions->SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
#ifdef ANSLPR_USE_CUDA
// Optionally add more execution providers via session_options
// E.g. for CUDA include cuda_provider_factory.h and uncomment the following line:
// nullptr for Status* indicates success
OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(*psessionOptions, 0);
//or status =nullptr; //if you don t have CUDA
if (status == nullptr) {
#endif //ANSLPR_USE_CUDA
Plates_types_classifier* onnx_net = nullptr;
#ifdef _WIN32
//step 4 declare an onnx session (ie model), by giving references to the runtime environment, session options and file path to the model
std::wstring widestr = std::wstring(model_filename.begin(), model_filename.end());
onnx_net = new Plates_types_classifier(*penv, widestr.c_str(), *psessionOptions, labels_filename);
#else
onnx_net = new Plates_types_classifier(*penv, model_filename.c_str(), *psessionOptions, labels_filename);
#endif
if (onnx_net != nullptr && penv != nullptr && psessionOptions != nullptr) {
std::unique_lock<std::mutex> lck(mtxMutex, std::defer_lock);
lck.lock();
plates_types_envs.push_back(penv);
l_plates_types_classifier_sessionOptions.push_back(psessionOptions);
plates_types_classifiers.push_back(onnx_net);
unsigned int id = GetNewId(plates_types_classifier_ids);
plates_types_classifier_ids.push_back(id);
lck.unlock();
return id;
}
else {
this->_logger->LogError("ANSALPR::InitPlatesClassifer. Error while creating onnxruntime session with file.", model_filename, __FILE__, __LINE__);
return 0;
}
#ifdef ANSLPR_USE_CUDA
}
else {
CheckStatus(status);
this->_logger->LogError("ANSALPR::InitPlatesClassifer.", "Cuda error", __FILE__, __LINE__);
return 0;
}
#endif //ANSLPR_USE_CUDA
}
else {
this->_logger->LogError("ANSALPR::InitPlatesClassifer.", "Error while creating SessionOptions", __FILE__, __LINE__);
return 0;
}
}
else {
this->_logger->LogError("ANSALPR::InitPlatesClassifer.", "Error while creating session environment (Ort::Env)", __FILE__, __LINE__);
return 0;
}
}
bool ANSALPR_RT::TwoStageALPR(const int width,//width of image
const int height,//height of image i.e. the specified dimensions of the image
const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
void* pbData, unsigned int step,// source image bytes buffer
unsigned int id_global_view,
unsigned int id_focused_on_lp,
unsigned int lpn_len,
char* lpn, cv::Rect& bbox)
{
if ((pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr) {
this->_logger->LogError("ANSALPR::TwoStageLPR.", "Condition on image (pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr not met", __FILE__, __LINE__);
return false;
}
else {
cv::Mat destMat;
if (pixOpt == 1)
{
destMat = cv::Mat(height, width, CV_8UC1, pbData, step);
}
if (pixOpt == 3)
{
destMat = cv::Mat(height, width, CV_8UC3, pbData, step);
}
if (pixOpt == 4)
{
destMat = cv::Mat(height, width, CV_8UC4, pbData, step);
}
std::list<Yolov5_alpr_onxx_detector*>::const_iterator it_global_view = GetDetector(id_global_view, detectors, detectors_ids);
if (it_global_view != detectors.end()) {
std::list<Yolov5_alpr_onxx_detector*>::const_iterator it_focused_on_lp = GetDetector(id_focused_on_lp, detectors, detectors_ids);
std::string lpn_str;
std::list<cv::Rect> ROIs;
std::unique_lock<std::mutex> lck(mtxMutex, std::defer_lock);
if (it_focused_on_lp != detectors.end()) {
lck.lock();
//for normal plates
ROIs = (*it_global_view)->TwoStage_LPR(*(*it_focused_on_lp), destMat, lpn_str);
//for small plates
lck.unlock();
}
else {
this->_logger->LogError("ANSALPR::TwoStageLPR.", "id_focused_on_lp does not point to a valid detector", __FILE__, __LINE__);
lck.lock();
//for normal plates
ROIs = (*it_global_view)->TwoStage_LPR(*(*it_global_view), destMat, lpn_str);
//for small plates
lck.unlock();
}
std::string::const_iterator it_lpn(lpn_str.begin());
int i = 0;
while (it_lpn != lpn_str.end() && i < lpn_len - 1) {
lpn[i] = *it_lpn;
i++; it_lpn++;
}
while (i < lpn_len) {
lpn[i] = '\0';
i++;
}
bbox = GetGlobalROI(ROIs);
return (lpn_str.length() > 0);
}
else {
this->_logger->LogError("ANSALPR::TwoStageLPR.", "id_global_view does not point to a valid detector", __FILE__, __LINE__);
return false;
}
}
}
cv::Rect ANSALPR_RT::GetGlobalROI(std::list<cv::Rect> ROIs) {
cv::Rect result;
if (ROIs.size() > 0) result = ROIs.front();
return result;
/*std::list<cv::Rect>::iterator it;
cv::Rect v = ROIs.front();
int x_min, y_min, w_min, h_min;
int x_max, y_max, w_max, h_max;
x_min = v.x;
x_max = v.x;
y_min = v.y;
y_max = v.y;
w_min = v.width;
w_max = v.width;
h_min = v.height;
h_max = v.height;
for (it = ROIs.begin(); it != ROIs.end(); ++it) {
if (x_min > it->x) { x_min = it->x; }
if (x_max < it->x) { x_max = it->x; }
if (y_min > it->y) { y_min = it->y; }
if (y_max < it->y) { y_max = it->y; }
if (w_min > it->width) { w_min = it->width; }
if (w_max < it->width) { w_max = it->width; }
if (h_min > it->height) { w_min = it->height; }
if (h_max < it->height) { w_max = it->height; }
}
v.x = x_min - 2;
v.y = y_min - 2;
v.width = x_max + w_max - x_min + 2;
v.height = h_max + 2;
return v;*/
}
bool ANSALPR_RT::TwoStageLPRPlatesTypeDetection(const int width,//width of image
const int height,//height of image i.e. the specified dimensions of the image
const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
void* pbData, unsigned int step,// source image bytes buffer
unsigned int id_global_view,
unsigned int id_focused_on_lp,
unsigned int id_plates_types_classifier,
unsigned int lpn_len, char* lpn, cv::Rect& bbox)
{
if ((pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr) {
this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "Condition on image (pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr not met", __FILE__, __LINE__);
return false;
}
else {
cv::Mat destMat;
if (pixOpt == 1)
{
destMat = cv::Mat(height, width, CV_8UC1, pbData, step);
}
if (pixOpt == 3)
{
destMat = cv::Mat(height, width, CV_8UC3, pbData, step);
}
if (pixOpt == 4)
{
destMat = cv::Mat(height, width, CV_8UC4, pbData, step);
}
std::list<Plates_types_classifier*>::const_iterator it_plates_types_classifier = GetPlatesTypesClassifier(id_plates_types_classifier,
plates_types_classifiers, plates_types_classifier_ids);
if (it_plates_types_classifier != plates_types_classifiers.end()) {
std::list<Yolov5_alpr_onxx_detector*>::const_iterator it_global_view = GetDetector(id_global_view, detectors, detectors_ids);
if (it_global_view != detectors.end()) {
std::list<Yolov5_alpr_onxx_detector*>::const_iterator it_focused_on_lp = GetDetector(id_focused_on_lp, detectors, detectors_ids);
std::string lpn_str;
std::list<cv::Rect> ROIs;
std::unique_lock<std::mutex> lck(mtxMutex, std::defer_lock);
if (it_focused_on_lp != detectors.end()) {
lck.lock();
//for normal plates
ROIs = (*it_global_view)->TwoStageLPR(*(*it_focused_on_lp), *(*it_plates_types_classifier), destMat, lpn_str);
//for small plates
lck.unlock();
}
else {
this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "id_focused_on_lp does not point to a valid detector", __FILE__, __LINE__);
lck.lock();
//for normal plates
ROIs = (*it_global_view)->TwoStageLPR(*(*it_global_view), *(*it_plates_types_classifier), destMat, lpn_str);
//for small plates
lck.unlock();
}
std::string::const_iterator it_lpn(lpn_str.begin());
int i = 0;
while (it_lpn != lpn_str.end() && i < lpn_len - 1) {
lpn[i] = *it_lpn;
i++; it_lpn++;
}
while (i < lpn_len) {
lpn[i] = '\0';
i++;
}
bbox = GetGlobalROI(ROIs);
return (lpn_str.length() > 0);
}
else {
this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "id_global_view does not point to a valid detector", __FILE__, __LINE__);
return false;
}
}
else {
this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "id_plates_types_classifier does not point to a valid detector", __FILE__, __LINE__);
return TwoStageALPR(width,//width of image
height,//height of image i.e. the specified dimensions of the image
pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
pbData, step,// source image bytes buffer
id_global_view, id_focused_on_lp, lpn_len, lpn, bbox);
}
}
}
bool ANSALPR_RT::CloseDetector(unsigned int id)
{
assert(detectors_ids.size() == detectors.size());
std::unique_lock<std::mutex> lck(mtxMutex, std::defer_lock);
lck.lock();
bool session_closed = CloseDetector(id, detectors_envs, l_detectors_sessionOptions, detectors, detectors_ids);
lck.unlock();
return session_closed;
}
bool ANSALPR_RT::ClosePlatesTypesClassifier(unsigned int id)
{
assert(plates_types_classifier_ids.size() == plates_types_classifiers.size());
std::unique_lock<std::mutex> lck(mtxMutex, std::defer_lock);
lck.lock();
bool session_closed = CloseDetector(id, plates_types_envs, l_plates_types_classifier_sessionOptions, plates_types_classifiers, plates_types_classifier_ids);
lck.unlock();
return session_closed;
}
}

97
modules/ANSLPR/ANSLPR_RT.h
View File

@@ -1,97 +0,0 @@
#ifndef ANSLPRRT_H
#define ANSLPRRT_H
#pragma once
#include "ANSLPR.h"
#include <onnxruntime_c_api.h>
#include <onnxruntime_cxx_api.h>
#include "yolov5_alpr_onnx_detector.h"
#include "ONNX_detector.h"
#include <thread> // std::thread
#include <mutex> // std::mutex, std::unique_lock, std::defer_lock
namespace ANSCENTER
{
class ANSLPR_API ANSALPR_RT :public ANSALPR {
private:
std::mutex mtxMutex; // mutex for critical section
std::string _globalModelFileName{};
std::string _focusedOnLPRModelFileName{};
std::string _platesTypesClassifierFileName{};
std::string _platesTypesLabelsFileName{};
size_t _globalViewId;
size_t _focusedOnLPRId;
size_t _platesTypesClassifierId;
[[nodiscard]] bool CloseReferences();
public:
ANSALPR_RT();
~ANSALPR_RT();
[[nodiscard]] bool Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) override;
[[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult);
[[nodiscard]] bool Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult);
[[nodiscard]] bool Destroy() override;
private:
[[nodiscard]] std::string VectorDetectionToJsonString(const std::vector<Object>& dets);
void CheckStatus(OrtStatus* status);
cv::Rect GetGlobalROI(std::list<cv::Rect> ROIs);
std::list<Yolov5_alpr_onxx_detector*>::const_iterator GetDetector(unsigned int id, const std::list<Yolov5_alpr_onxx_detector*>& detectors,
const std::list<unsigned int>& detectors_ids);
std::list<Plates_types_classifier*>::const_iterator GetPlatesTypesClassifier(unsigned int id, const std::list<Plates_types_classifier*>& plates_types_classifiers,
const std::list<unsigned int>& plates_types_classifiers_ids);
unsigned int GetNewId(const std::list<unsigned int>& detectors_ids);
bool CloseDetector(unsigned int id, std::list<Ort::Env*>& _envs, std::list<Ort::SessionOptions*>& _lsessionOptions, std::list<Yolov5_alpr_onxx_detector*>& _detectors,
std::list<unsigned int>& _detectors_ids);
bool CloseDetector(unsigned int id, std::list<Ort::Env*>& _envs, std::list<Ort::SessionOptions*>& _lsessionOptions, std::list<Plates_types_classifier*>& _detectors,
std::list<unsigned int>& _detectors_ids);
unsigned int InitYoloDetector(unsigned int len, const char* model_file);
unsigned int InitPlatesClassifer(unsigned int len_models_filename, const char* model_file, unsigned int len_labels_filename, const char* labels_file);
bool TwoStageALPR(const int width,//width of image
const int height,//height of image i.e. the specified dimensions of the image
const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
void* pbData, unsigned int step,// source image bytes buffer
unsigned int id_global_view,
unsigned int id_focused_on_lp,
unsigned int lpn_len, char* lpn, cv::Rect& bbox);
bool TwoStageLPRPlatesTypeDetection(const int width,//width of image
const int height,//height of image i.e. the specified dimensions of the image
const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image)
void* pbData,
unsigned int step,// source image bytes buffer
unsigned int id_global_view,
unsigned int id_focused_on_lp,
unsigned int id_plates_types_classifier,
unsigned int lpn_len,
char* lpn,
cv::Rect& bbox);
bool CloseDetector(unsigned int id);
bool ClosePlatesTypesClassifier(unsigned int id);
private:
// Version-negotiated OrtApi pointer: caps at the DLL's max supported API level so
// that a newer SDK header (ORT_API_VERSION=22) paired with an older runtime DLL
// (e.g. ORT 1.17.1, API ≤17) does not emit "[ORT ERROR] API version not available".
// Mirrors the negotiation pattern used in EPLoader.cpp.
const OrtApi* g_ort = []() -> const OrtApi* {
const OrtApiBase* base = OrtGetApiBase();
int dllMaxApi = ORT_API_VERSION;
const char* verStr = base->GetVersionString();
int major = 0, minor = 0;
if (verStr && sscanf(verStr, "%d.%d", &major, &minor) == 2)
dllMaxApi = minor;
int targetApi = (ORT_API_VERSION < dllMaxApi) ? ORT_API_VERSION : dllMaxApi;
return base->GetApi(targetApi);
}();
std::list<Ort::Env*> detectors_envs;
std::list<Ort::SessionOptions*> l_detectors_sessionOptions;
std::list<Yolov5_alpr_onxx_detector*> detectors;
std::list<unsigned int> detectors_ids;
std::list<Ort::Env*> plates_types_envs;
std::list<Ort::SessionOptions*> l_plates_types_classifier_sessionOptions;
std::list<Plates_types_classifier*> plates_types_classifiers;
std::list<unsigned int> plates_types_classifier_ids;
};
}
#endif

2
modules/ANSLPR/CMakeLists.txt
View File

@@ -4,6 +4,7 @@ set(ANSLPR_SOURCES
ANSLPR.cpp
ANSLPR_CPU.cpp
ANSLPR_OD.cpp
ANSLPR_OCR.cpp
ANSGpuFrameRegistry.cpp
GpuNV12SlotPool.cpp
dllmain.cpp
@@ -56,6 +57,7 @@ target_link_libraries(ANSLPR
PRIVATE ANSODEngine
PUBLIC ANSLicensingSystem # PUBLIC: Utility.h/SPDLogger symbols must be re-exported
PRIVATE ANSMOT
PRIVATE ANSOCR # ANSALPR_OCR uses ANSONNXOCR from the OCR module
PRIVATE labview
PRIVATE spdlog_dep
PRIVATE opencv

15
modules/ANSLPR/dllmain.cpp
View File

@@ -4,6 +4,7 @@
#include "ANSLPR.h"
#include "ANSLPR_CPU.h"
#include "ANSLPR_OD.h"
#include "ANSLPR_OCR.h"
#include "ANSLibsLoader.h"
#include "ANSGpuFrameRegistry.h" // gpu_frame_lookup(cv::Mat*)
#include <unordered_set>
@@ -141,6 +142,9 @@ static int CreateANSALPRHandle_Impl(ANSCENTER::ANSALPR** Handle, const char* lic
else if (engineType == 1) {
(*Handle) = new ANSCENTER::ANSALPR_OD();
}
else if (engineType == 2) {
(*Handle) = new ANSCENTER::ANSALPR_OCR();// ONNX OCR (PaddleOCR v5)
}
else {
return 0;
}
@@ -757,6 +761,17 @@ extern "C" ANSLPR_API int ANSALPR_SetALPRCheckerEnabled(ANSCENTER::ANSALPR**
}
}
extern "C" ANSLPR_API int ANSALPR_SetCountry(ANSCENTER::ANSALPR** Handle, int country) {
if (!Handle || !*Handle) return -1;
try {
(*Handle)->SetCountry(static_cast<ANSCENTER::Country>(country));
return 1;
}
catch (...) {
return 0;
}
}
extern "C" ANSLPR_API int ANSALPR_GetFormats(ANSCENTER::ANSALPR** Handle, LStrHandle Lstrformats)// semi separated formats
{
if (!Handle || !*Handle) return -1;