843 lines
42 KiB
C++
843 lines
42 KiB
C++
#include "ANSLPR_OV.h"
|
|
namespace ANSCENTER {
|
|
|
|
void tryPush(const std::weak_ptr<Worker>& worker, std::shared_ptr<Task>&& task) {
|
|
try {
|
|
std::shared_ptr<Worker>(worker)->push(task);
|
|
} catch (const std::bad_weak_ptr&) {}
|
|
}
|
|
|
|
void fillROIColor(cv::Mat& displayImage, cv::Rect roi, cv::Scalar color, double opacity) {
|
|
if (opacity > 0) {
|
|
roi = roi & cv::Rect(0, 0, displayImage.cols, displayImage.rows);
|
|
cv::Mat textROI = displayImage(roi);
|
|
cv::addWeighted(color, opacity, textROI, 1.0 - opacity , 0.0, textROI);
|
|
}
|
|
}
|
|
|
|
void putTextOnImage(cv::Mat& displayImage, std::string str, cv::Point p,
|
|
cv::HersheyFonts font, double fontScale, cv::Scalar color,
|
|
int thickness = 1, cv::Scalar bgcolor = cv::Scalar(),
|
|
double opacity = 0) {
|
|
int baseline = 0;
|
|
cv::Size textSize = cv::getTextSize(str, font, 0.5, 1, &baseline);
|
|
fillROIColor(displayImage, cv::Rect(cv::Point(p.x, p.y + baseline),
|
|
cv::Point(p.x + textSize.width, p.y - textSize.height)),
|
|
bgcolor, opacity);
|
|
cv::putText(displayImage, str, p, font, fontScale, color, thickness);
|
|
}
|
|
|
|
Detector::Detector(ov::Core& core,
|
|
const std::string& deviceName,
|
|
const std::string& xmlPath,
|
|
const std::vector<float>& detectionTresholds,
|
|
const bool autoResize) :
|
|
m_autoResize(autoResize),
|
|
m_detectionTresholds{ detectionTresholds }
|
|
{
|
|
slog::info << "Reading model: " << xmlPath << slog::endl;
|
|
std::shared_ptr<ov::Model> model = core.read_model(xmlPath);
|
|
logBasicModelInfo(model);
|
|
// Check model inputs and outputs
|
|
ov::OutputVector inputs = model->inputs();
|
|
if (inputs.size() != 1) {
|
|
throw std::logic_error("Detector should have only one input");
|
|
}
|
|
|
|
m_detectorInputName = model->input().get_any_name();
|
|
|
|
ov::Layout modelLayout = ov::layout::get_layout(model->input());
|
|
if (modelLayout.empty())
|
|
modelLayout = { "NCHW" };
|
|
|
|
ov::OutputVector outputs = model->outputs();
|
|
if (outputs.size() != 1) {
|
|
throw std::logic_error("Vehicle Detection network should have only one output");
|
|
}
|
|
|
|
ov::Output<ov::Node> output = outputs[0];
|
|
|
|
m_detectorOutputName = output.get_any_name();
|
|
ov::Shape output_shape = output.get_shape();
|
|
|
|
if (output_shape.size() != 4) {
|
|
throw std::logic_error("Incorrect output dimensions for SSD");
|
|
}
|
|
|
|
if (maxProposalCount != output_shape[2]) {
|
|
throw std::logic_error("unexpected ProposalCount");
|
|
}
|
|
if (objectSize != output_shape[3]) {
|
|
throw std::logic_error("Output should have 7 as a last dimension");
|
|
}
|
|
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
|
ov::preprocess::InputInfo& inputInfo = ppp.input();
|
|
ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor();
|
|
// configure desired input type and layout, the
|
|
// use preprocessor to convert to actual model input type and layout
|
|
inputTensorInfo.set_element_type(ov::element::u8);
|
|
inputTensorInfo.set_layout({ "NHWC" });
|
|
if (autoResize) {
|
|
inputTensorInfo.set_spatial_dynamic_shape();
|
|
}
|
|
|
|
ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model();
|
|
inputModelInfo.set_layout(modelLayout);
|
|
|
|
ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess();
|
|
preProcessSteps.convert_layout(modelLayout);
|
|
preProcessSteps.convert_element_type(ov::element::f32);
|
|
if (autoResize) {
|
|
preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
|
|
}
|
|
|
|
model = ppp.build();
|
|
|
|
slog::info << "Preprocessor configuration: " << slog::endl;
|
|
slog::info << ppp << slog::endl;
|
|
m_compiled_model = core.compile_model(model, deviceName);
|
|
logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "Vehicle And License Plate Detection");
|
|
}
|
|
|
|
ov::InferRequest Detector::createInferRequest() {
|
|
return m_compiled_model.create_infer_request();
|
|
}
|
|
void Detector::setImage(ov::InferRequest& inferRequest, const cv::Mat& img) {
|
|
ov::Tensor inputTensor = inferRequest.get_tensor(m_detectorInputName);
|
|
ov::Shape shape = inputTensor.get_shape();
|
|
if (m_autoResize) {
|
|
if (!img.isSubmatrix()) {
|
|
// just wrap Mat object with Tensor without additional memory allocation
|
|
ov::Tensor frameTensor = wrapMat2Tensor(img);
|
|
inferRequest.set_tensor(m_detectorInputName, frameTensor);
|
|
}
|
|
else {
|
|
throw std::logic_error("Sparse matrix are not supported");
|
|
}
|
|
}
|
|
else {
|
|
// resize and copy data from image to tensor using OpenCV
|
|
resize2tensor(img, inputTensor);
|
|
}
|
|
}
|
|
std::list<Detector::Result> Detector::getResults(ov::InferRequest& inferRequest,
|
|
cv::Size upscale,
|
|
std::vector<std::string>& rawResults) {
|
|
// there is no big difference if InferReq of detector from another device is passed
|
|
// because the processing is the same for the same topology
|
|
std::list<Result> results;
|
|
ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName);
|
|
const float* const detections = output_tensor.data<float>();
|
|
// pretty much regular SSD post-processing
|
|
for (int i = 0; i < maxProposalCount; i++) {
|
|
float image_id = detections[i * objectSize + 0]; // in case of batch
|
|
if (image_id < 0) { // indicates end of detections
|
|
break;
|
|
}
|
|
size_t label = static_cast<decltype(m_detectionTresholds.size())>(detections[i * objectSize + 1]);
|
|
float confidence = detections[i * objectSize + 2];
|
|
if (label - 1 < m_detectionTresholds.size() && confidence < m_detectionTresholds[label - 1]) {
|
|
continue;
|
|
}
|
|
|
|
cv::Rect rect;
|
|
rect.x = static_cast<int>(detections[i * objectSize + 3] * upscale.width);
|
|
rect.y = static_cast<int>(detections[i * objectSize + 4] * upscale.height);
|
|
rect.width = static_cast<int>(detections[i * objectSize + 5] * upscale.width) - rect.x;
|
|
rect.height = static_cast<int>(detections[i * objectSize + 6] * upscale.height) - rect.y;
|
|
results.push_back(Result{ label, confidence, rect });
|
|
std::ostringstream rawResultsStream;
|
|
rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence
|
|
<< " (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")";
|
|
rawResults.push_back(rawResultsStream.str());
|
|
}
|
|
return results;
|
|
}
|
|
|
|
VehicleAttributesClassifier::VehicleAttributesClassifier(ov::Core& core,
|
|
const std::string& deviceName,
|
|
const std::string& xmlPath,
|
|
const bool autoResize) :m_autoResize(autoResize)
|
|
{
|
|
slog::info << "Reading model: " << xmlPath << slog::endl;
|
|
std::shared_ptr<ov::Model> model = core.read_model(xmlPath);
|
|
logBasicModelInfo(model);
|
|
ov::OutputVector inputs = model->inputs();
|
|
if (inputs.size() != 1) {
|
|
throw std::logic_error("Vehicle Attribs topology should have only one input");
|
|
}
|
|
m_attributesInputName = model->input().get_any_name();
|
|
ov::Layout modelLayout = ov::layout::get_layout(model->input());
|
|
if (modelLayout.empty())
|
|
modelLayout = { "NCHW" };
|
|
|
|
ov::OutputVector outputs = model->outputs();
|
|
if (outputs.size() != 2) {
|
|
throw std::logic_error("Vehicle Attribs Network expects networks having two outputs");
|
|
}
|
|
|
|
// color is the first output
|
|
m_outputNameForColor = outputs[0].get_any_name();
|
|
// type is the second output.
|
|
m_outputNameForType = outputs[1].get_any_name();
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
|
ov::preprocess::InputInfo& inputInfo = ppp.input();
|
|
ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor();
|
|
// configure desired input type and layout, the
|
|
// use preprocessor to convert to actual model input type and layout
|
|
inputTensorInfo.set_element_type(ov::element::u8);
|
|
inputTensorInfo.set_layout({ "NHWC" });
|
|
if (autoResize) {
|
|
inputTensorInfo.set_spatial_dynamic_shape();
|
|
}
|
|
ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess();
|
|
preProcessSteps.convert_layout(modelLayout);
|
|
preProcessSteps.convert_element_type(ov::element::f32);
|
|
if (autoResize) {
|
|
preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
|
|
}
|
|
|
|
ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model();
|
|
inputModelInfo.set_layout(modelLayout);
|
|
|
|
model = ppp.build();
|
|
|
|
slog::info << "Preprocessor configuration: " << slog::endl;
|
|
slog::info << ppp << slog::endl;
|
|
|
|
m_compiled_model = core.compile_model(model, deviceName);
|
|
logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "Vehicle Attributes Recognition");
|
|
}
|
|
ov::InferRequest VehicleAttributesClassifier::createInferRequest() {
|
|
return m_compiled_model.create_infer_request();
|
|
}
|
|
void VehicleAttributesClassifier::setImage(ov::InferRequest& inferRequest,
|
|
const cv::Mat& img,
|
|
const cv::Rect vehicleRect)
|
|
{
|
|
ov::Tensor inputTensor = inferRequest.get_tensor(m_attributesInputName);
|
|
ov::Shape shape = inputTensor.get_shape();
|
|
if (m_autoResize) {
|
|
ov::Tensor frameTensor = wrapMat2Tensor(img);
|
|
ov::Coordinate p00({ 0, static_cast<size_t>(vehicleRect.y), static_cast<size_t>(vehicleRect.x), 0 });
|
|
ov::Coordinate p01({ 1, static_cast<size_t>(vehicleRect.y + vehicleRect.height), static_cast<size_t>(vehicleRect.x) + vehicleRect.width, 3 });
|
|
ov::Tensor roiTensor(frameTensor, p00, p01);
|
|
inferRequest.set_tensor(m_attributesInputName, roiTensor);
|
|
}
|
|
else {
|
|
const cv::Mat& vehicleImage = img(vehicleRect);
|
|
resize2tensor(vehicleImage, inputTensor);
|
|
}
|
|
}
|
|
std::pair<std::string, std::string> VehicleAttributesClassifier::getResults(ov::InferRequest& inferRequest) {
|
|
static const std::string colors[] = {
|
|
"white", "gray", "yellow", "red", "green", "blue", "black"
|
|
};
|
|
static const std::string types[] = {
|
|
"car", "van", "truck", "bus"
|
|
};
|
|
|
|
// 7 possible colors for each vehicle and we should select the one with the maximum probability
|
|
ov::Tensor colorsTensor = inferRequest.get_tensor(m_outputNameForColor);
|
|
const float* colorsValues = colorsTensor.data<float>();
|
|
|
|
// 4 possible types for each vehicle and we should select the one with the maximum probability
|
|
ov::Tensor typesTensor = inferRequest.get_tensor(m_outputNameForType);
|
|
const float* typesValues = typesTensor.data<float>();
|
|
|
|
const auto color_id = std::max_element(colorsValues, colorsValues + 7) - colorsValues;
|
|
const auto type_id = std::max_element(typesValues, typesValues + 4) - typesValues;
|
|
|
|
return std::pair<std::string, std::string>(colors[color_id], types[type_id]);
|
|
}
|
|
|
|
Lpr::Lpr(ov::Core& core,
|
|
const std::string& deviceName,
|
|
const std::string& xmlPath,
|
|
const bool autoResize) :m_autoResize(autoResize)
|
|
{
|
|
slog::info << "Reading model: " << xmlPath << slog::endl;
|
|
std::shared_ptr<ov::Model> model = core.read_model(xmlPath);
|
|
logBasicModelInfo(model);
|
|
// LPR network should have 2 inputs (and second is just a stub) and one output
|
|
// Check inputs
|
|
ov::OutputVector inputs = model->inputs();
|
|
if (inputs.size() != 1 && inputs.size() != 2) {
|
|
throw std::logic_error("LPR should have 1 or 2 inputs");
|
|
}
|
|
|
|
for (auto input : inputs) {
|
|
if (input.get_shape().size() == 4) {
|
|
m_LprInputName = input.get_any_name();
|
|
m_modelLayout = ov::layout::get_layout(input);
|
|
if (m_modelLayout.empty())
|
|
m_modelLayout = { "NCHW" };
|
|
}
|
|
// LPR model that converted from Caffe have second a stub input
|
|
if (input.get_shape().size() == 2)
|
|
m_LprInputSeqName = input.get_any_name();
|
|
}
|
|
|
|
// Check outputs
|
|
|
|
m_maxSequenceSizePerPlate = 1;
|
|
|
|
ov::OutputVector outputs = model->outputs();
|
|
if (outputs.size() != 1) {
|
|
throw std::logic_error("LPR should have 1 output");
|
|
}
|
|
|
|
m_LprOutputName = outputs[0].get_any_name();
|
|
|
|
for (size_t dim : outputs[0].get_shape()) {
|
|
if (dim == 1) {
|
|
continue;
|
|
}
|
|
if (m_maxSequenceSizePerPlate == 1) {
|
|
m_maxSequenceSizePerPlate = dim;
|
|
}
|
|
else {
|
|
throw std::logic_error("Every dimension of LPR output except for one must be of size 1");
|
|
}
|
|
}
|
|
|
|
ov::preprocess::PrePostProcessor ppp(model);
|
|
ov::preprocess::InputInfo& inputInfo = ppp.input(m_LprInputName);
|
|
ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor();
|
|
// configure desired input type and layout, the
|
|
// use preprocessor to convert to actual model input type and layout
|
|
inputTensorInfo.set_element_type(ov::element::u8);
|
|
inputTensorInfo.set_layout({ "NHWC" });
|
|
if (autoResize) {
|
|
inputTensorInfo.set_spatial_dynamic_shape();
|
|
}
|
|
|
|
ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess();
|
|
preProcessSteps.convert_layout(m_modelLayout);
|
|
preProcessSteps.convert_element_type(ov::element::f32);
|
|
if (autoResize) {
|
|
preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
|
|
}
|
|
|
|
ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model();
|
|
inputModelInfo.set_layout(m_modelLayout);
|
|
|
|
model = ppp.build();
|
|
|
|
slog::info << "Preprocessor configuration: " << slog::endl;
|
|
slog::info << ppp << slog::endl;
|
|
|
|
m_compiled_model = core.compile_model(model, deviceName);
|
|
logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "License Plate Recognition");
|
|
}
|
|
ov::InferRequest Lpr::createInferRequest() {
|
|
return m_compiled_model.create_infer_request();
|
|
}
|
|
void Lpr::setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect plateRect) {
|
|
ov::Tensor inputTensor = inferRequest.get_tensor(m_LprInputName);
|
|
ov::Shape shape = inputTensor.get_shape();
|
|
if ((shape.size() == 4) && m_autoResize) {
|
|
// autoResize is set
|
|
ov::Tensor frameTensor = wrapMat2Tensor(img);
|
|
ov::Coordinate p00({ 0, static_cast<size_t>(plateRect.y), static_cast<size_t>(plateRect.x), 0 });
|
|
ov::Coordinate p01({ 1, static_cast<size_t>(plateRect.y + plateRect.height), static_cast<size_t>(plateRect.x + plateRect.width), 3 });
|
|
ov::Tensor roiTensor(frameTensor, p00, p01);
|
|
inferRequest.set_tensor(m_LprInputName, roiTensor);
|
|
}
|
|
else {
|
|
const cv::Mat& vehicleImage = img(plateRect);
|
|
resize2tensor(vehicleImage, inputTensor);
|
|
}
|
|
|
|
if (m_LprInputSeqName != "") {
|
|
ov::Tensor inputSeqTensor = inferRequest.get_tensor(m_LprInputSeqName);
|
|
float* data = inputSeqTensor.data<float>();
|
|
std::fill(data, data + inputSeqTensor.get_shape()[0], 1.0f);
|
|
}
|
|
}
|
|
std::string Lpr::getResults(ov::InferRequest& inferRequest) {
|
|
static const char* const items[] = {
|
|
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "", "", "",
|
|
"", "",
|
|
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J",
|
|
"K", "L", "M", "N", "O", "P", "Q", "R", "S", "T",
|
|
"U", "V", "W", "X", "Y", "Z"
|
|
};
|
|
std::string result;
|
|
result.reserve(14u + 6u); // the longest province name + 6 plate signs
|
|
|
|
ov::Tensor lprOutputTensor = inferRequest.get_tensor(m_LprOutputName);
|
|
ov::element::Type precision = lprOutputTensor.get_element_type();
|
|
|
|
// up to 88 items per license plate, ended with "-1"
|
|
switch (precision) {
|
|
case ov::element::i32:
|
|
{
|
|
const auto data = lprOutputTensor.data<int32_t>();
|
|
for (int i = 0; i < m_maxSequenceSizePerPlate; i++) {
|
|
int32_t val = data[i];
|
|
if (val == -1) {
|
|
break;
|
|
}
|
|
result += items[val];
|
|
}
|
|
}
|
|
break;
|
|
|
|
case ov::element::f32:
|
|
{
|
|
const auto data = lprOutputTensor.data<float>();
|
|
for (int i = 0; i < m_maxSequenceSizePerPlate; i++) {
|
|
int32_t val = int32_t(data[i]);
|
|
if (val == -1) {
|
|
break;
|
|
}
|
|
result += items[val];
|
|
}
|
|
}
|
|
break;
|
|
|
|
default:
|
|
throw std::logic_error("Not expected output blob precision");
|
|
break;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Utilities
|
|
ReborningVideoFrame::~ReborningVideoFrame() {
|
|
try {
|
|
const std::shared_ptr<Worker>& worker = std::shared_ptr<Worker>(context.readersContext.readersWorker);
|
|
context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock();
|
|
const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID];
|
|
context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock();
|
|
std::shared_ptr<ReborningVideoFrame> reborn = std::make_shared<ReborningVideoFrame>(context, sourceID, frameId, frame);
|
|
worker->push(std::make_shared<Reader>(reborn));
|
|
}
|
|
catch (const std::bad_weak_ptr&) {}
|
|
}
|
|
|
|
void ResAggregator::process() {
|
|
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
|
|
context.freeDetectionInfersCount += context.detectorsInfers.inferRequests.lockedSize();
|
|
context.frameCounter++;
|
|
context.boxesAndDescrs = boxesAndDescrs;
|
|
try {
|
|
std::shared_ptr<Worker>(context.resAggregatorsWorker)->stop();
|
|
}
|
|
catch (const std::bad_weak_ptr&) {}
|
|
}
|
|
|
|
bool DetectionsProcessor::isReady() {
|
|
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
|
|
if (requireGettingNumberOfDetections) {
|
|
classifiersAggregator = std::make_shared<ClassifiersAggregator>(sharedVideoFrame);
|
|
std::list<Detector::Result> results;
|
|
results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections);
|
|
for (Detector::Result result : results) {
|
|
switch (result.label) {
|
|
case 1:// Vehicle
|
|
{
|
|
vehicleRects.emplace_back(result.location & cv::Rect{ cv::Point(0, 0), sharedVideoFrame->frame.size() });
|
|
break;
|
|
}
|
|
case 2:// License Plate
|
|
{
|
|
// expanding a bounding box a bit, better for the license plate recognition
|
|
result.location.x -= 5;
|
|
result.location.y -= 5;
|
|
result.location.width += 10;
|
|
result.location.height += 10;
|
|
plateRects.emplace_back(result.location & cv::Rect{ cv::Point(0, 0), sharedVideoFrame->frame.size() });
|
|
break;
|
|
}
|
|
default:
|
|
throw std::runtime_error("Unexpected detection results"); // must never happen
|
|
break;
|
|
}
|
|
}
|
|
context.detectorsInfers.inferRequests.lockedPushBack(*inferRequest);
|
|
requireGettingNumberOfDetections = false;
|
|
}
|
|
|
|
if ((vehicleRects.empty()) && (plateRects.empty())) {
|
|
return true;
|
|
}
|
|
else {
|
|
InferRequestsContainer& attributesInfers = context.attributesInfers;
|
|
attributesInfers.inferRequests.mutex.lock();
|
|
const std::size_t numberOfAttributesInferRequestsAcquired = std::min(vehicleRects.size(), attributesInfers.inferRequests.container.size());
|
|
reservedAttributesRequests.assign(attributesInfers.inferRequests.container.end() - numberOfAttributesInferRequestsAcquired,attributesInfers.inferRequests.container.end());
|
|
attributesInfers.inferRequests.container.erase(attributesInfers.inferRequests.container.end() - numberOfAttributesInferRequestsAcquired,attributesInfers.inferRequests.container.end());
|
|
attributesInfers.inferRequests.mutex.unlock();
|
|
|
|
InferRequestsContainer& platesInfers = context.platesInfers;
|
|
platesInfers.inferRequests.mutex.lock();
|
|
const std::size_t numberOfLprInferRequestsAcquired = std::min(plateRects.size(), platesInfers.inferRequests.container.size());
|
|
reservedLprRequests.assign(platesInfers.inferRequests.container.end() - numberOfLprInferRequestsAcquired, platesInfers.inferRequests.container.end());
|
|
platesInfers.inferRequests.container.erase(platesInfers.inferRequests.container.end() - numberOfLprInferRequestsAcquired,platesInfers.inferRequests.container.end());
|
|
platesInfers.inferRequests.mutex.unlock();
|
|
return numberOfAttributesInferRequestsAcquired || numberOfLprInferRequestsAcquired;
|
|
}
|
|
}
|
|
|
|
void DetectionsProcessor::process() {
|
|
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
|
|
auto vehicleRectsIt = vehicleRects.begin();
|
|
for (auto attributesRequestIt = reservedAttributesRequests.begin(); attributesRequestIt != reservedAttributesRequests.end();
|
|
vehicleRectsIt++, attributesRequestIt++) {
|
|
const cv::Rect vehicleRect = *vehicleRectsIt;
|
|
ov::InferRequest& attributesRequest = *attributesRequestIt;
|
|
context.detectionsProcessorsContext.vehicleAttributesClassifier.setImage(attributesRequest, sharedVideoFrame->frame, vehicleRect);
|
|
|
|
attributesRequest.set_callback(
|
|
std::bind(
|
|
[](std::shared_ptr<ClassifiersAggregator> classifiersAggregator,
|
|
ov::InferRequest& attributesRequest,
|
|
cv::Rect rect,
|
|
Context& context) {
|
|
attributesRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object
|
|
const std::pair<std::string, std::string>& attributes =context.detectionsProcessorsContext.vehicleAttributesClassifier.getResults(attributesRequest);
|
|
if (((classifiersAggregator->sharedVideoFrame->frameId == 0 && !context.isVideo) || context.isVideo)) {
|
|
classifiersAggregator->rawAttributes.lockedPushBack("Vehicle Attributes results:" + attributes.first + ';' + attributes.second);
|
|
}
|
|
classifiersAggregator->push(BboxAndDescr{ BboxAndDescr::ObjectType::VEHICLE, rect, attributes.first + ' ' + attributes.second });
|
|
context.attributesInfers.inferRequests.lockedPushBack(attributesRequest);
|
|
},
|
|
classifiersAggregator,
|
|
std::ref(attributesRequest),
|
|
vehicleRect,
|
|
std::ref(context)));
|
|
attributesRequest.start_async();
|
|
}
|
|
vehicleRects.erase(vehicleRects.begin(), vehicleRectsIt);
|
|
|
|
auto plateRectsIt = plateRects.begin();
|
|
for (auto lprRequestsIt = reservedLprRequests.begin(); lprRequestsIt != reservedLprRequests.end(); plateRectsIt++, lprRequestsIt++) {
|
|
const cv::Rect plateRect = *plateRectsIt;
|
|
ov::InferRequest& lprRequest = *lprRequestsIt;
|
|
context.detectionsProcessorsContext.lpr.setImage(lprRequest, sharedVideoFrame->frame, plateRect);
|
|
|
|
lprRequest.set_callback(
|
|
std::bind(
|
|
[](std::shared_ptr<ClassifiersAggregator> classifiersAggregator,
|
|
ov::InferRequest& lprRequest,
|
|
cv::Rect rect,
|
|
Context& context) {
|
|
lprRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object
|
|
|
|
std::string result = context.detectionsProcessorsContext.lpr.getResults(lprRequest);
|
|
|
|
if (((classifiersAggregator->sharedVideoFrame->frameId == 0 && !context.isVideo) || context.isVideo)) {
|
|
classifiersAggregator->rawDecodedPlates.lockedPushBack("License Plate Recognition results:" + result);
|
|
}
|
|
classifiersAggregator->push(BboxAndDescr{ BboxAndDescr::ObjectType::PLATE, rect, std::move(result) });
|
|
context.platesInfers.inferRequests.lockedPushBack(lprRequest);
|
|
}, classifiersAggregator,
|
|
std::ref(lprRequest),
|
|
plateRect,
|
|
std::ref(context)));
|
|
|
|
lprRequest.start_async();
|
|
}
|
|
plateRects.erase(plateRects.begin(), plateRectsIt);
|
|
|
|
if (!vehicleRects.empty() || !plateRects.empty()) {
|
|
tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker,
|
|
std::make_shared<DetectionsProcessor>(sharedVideoFrame, std::move(classifiersAggregator), std::move(vehicleRects), std::move(plateRects)));
|
|
}
|
|
}
|
|
|
|
bool InferTask::isReady() {
|
|
InferRequestsContainer& detectorsInfers = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context.detectorsInfers;
|
|
if (detectorsInfers.inferRequests.container.empty()) {
|
|
return false;
|
|
}
|
|
else {
|
|
detectorsInfers.inferRequests.mutex.lock();
|
|
if (detectorsInfers.inferRequests.container.empty()) {
|
|
detectorsInfers.inferRequests.mutex.unlock();
|
|
return false;
|
|
}
|
|
else {
|
|
return true; // process() will unlock the mutex
|
|
}
|
|
}
|
|
}
|
|
|
|
void InferTask::process() {
|
|
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
|
|
InferRequestsContainer& detectorsInfers = context.detectorsInfers;
|
|
std::reference_wrapper<ov::InferRequest> inferRequest = detectorsInfers.inferRequests.container.back();
|
|
detectorsInfers.inferRequests.container.pop_back();
|
|
detectorsInfers.inferRequests.mutex.unlock();
|
|
context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame);
|
|
inferRequest.get().set_callback(
|
|
std::bind(
|
|
[](VideoFrame::Ptr sharedVideoFrame,
|
|
ov::InferRequest& inferRequest,
|
|
Context& context) {
|
|
inferRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object
|
|
tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker,
|
|
std::make_shared<DetectionsProcessor>(sharedVideoFrame, &inferRequest));
|
|
}, sharedVideoFrame,
|
|
inferRequest,
|
|
std::ref(context)));
|
|
|
|
inferRequest.get().start_async();
|
|
// do not push as callback does it
|
|
}
|
|
|
|
bool Reader::isReady() {
|
|
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
|
|
context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].lock();
|
|
if (context.readersContext.lastCapturedFrameIds[sharedVideoFrame->sourceID] + 1 == sharedVideoFrame->frameId) {
|
|
return true;
|
|
}
|
|
else {
|
|
context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].unlock();
|
|
return false;
|
|
}
|
|
}
|
|
void Reader::process() {
|
|
unsigned sourceID = sharedVideoFrame->sourceID;
|
|
sharedVideoFrame->timestamp = std::chrono::steady_clock::now();
|
|
Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
|
|
const std::vector<std::shared_ptr<InputChannel>>& inputChannels = context.readersContext.inputChannels;
|
|
if (inputChannels[sourceID]->read(sharedVideoFrame->frame)) {
|
|
context.readersContext.lastCapturedFrameIds[sourceID]++;
|
|
context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock();
|
|
tryPush(context.inferTasksContext.inferTasksWorker, std::make_shared<InferTask>(sharedVideoFrame));
|
|
}
|
|
else {
|
|
context.readersContext.lastCapturedFrameIds[sourceID]++;
|
|
context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock();
|
|
try {
|
|
std::shared_ptr<Worker>(context.resAggregatorsWorker)->stop();
|
|
}
|
|
catch (const std::bad_weak_ptr&) {}
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// Main class
|
|
/// </summary>
|
|
ANSALPR_OV::ANSALPR_OV() {};
|
|
ANSALPR_OV::~ANSALPR_OV() {
|
|
if (_detector == nullptr) {
|
|
delete _detector;
|
|
_detector = nullptr;
|
|
}
|
|
if (_vehicleAttributesClassifier == nullptr) {
|
|
delete _vehicleAttributesClassifier;
|
|
_vehicleAttributesClassifier = nullptr;
|
|
}
|
|
if (_lpr == nullptr) {
|
|
delete _lpr;
|
|
_lpr = nullptr;
|
|
}
|
|
};
|
|
bool ANSALPR_OV::Destroy() {
|
|
if (_detector == nullptr) {
|
|
delete _detector;
|
|
_detector = nullptr;
|
|
}
|
|
if (_vehicleAttributesClassifier == nullptr) {
|
|
delete _vehicleAttributesClassifier;
|
|
_vehicleAttributesClassifier = nullptr;
|
|
}
|
|
if (_lpr == nullptr) {
|
|
delete _lpr;
|
|
_lpr = nullptr;
|
|
}
|
|
return true;
|
|
};
|
|
bool ANSALPR_OV::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) {
|
|
try {
|
|
_licenseKey = licenseKey;
|
|
_licenseValid = false;
|
|
CheckLicense();
|
|
if (!_licenseValid) {
|
|
this->_logger->LogError("ANSALPR_OV::Initialize.", "License is not valid.", __FILE__, __LINE__);
|
|
return false;
|
|
}
|
|
// Extract model folder
|
|
// 0. Check if the modelZipFilePath exist?
|
|
if (!FileExist(modelZipFilePath)) {
|
|
this->_logger->LogFatal("ANSALPR_OV::Initialize", "Model zip file is not exist", __FILE__, __LINE__);
|
|
}
|
|
// 1. Unzip model zip file to a special location with folder name as model file (and version)
|
|
std::string outputFolder;
|
|
std::vector<std::string> passwordArray;
|
|
if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword);
|
|
passwordArray.push_back("AnsDemoModels20@!");
|
|
passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag==");
|
|
passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg==");
|
|
std::string modelName = GetFileNameWithoutExtension(modelZipFilePath);
|
|
|
|
size_t vectorSize = passwordArray.size();
|
|
for (size_t i = 0; i < vectorSize; i++) {
|
|
if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false))
|
|
break; // Break the loop when the condition is met.
|
|
}
|
|
// 2. Check if the outputFolder exist
|
|
if (!FolderExist(_modelFolder)) {
|
|
this->_logger->LogError("ANSALPR_OV::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__);
|
|
return false; // That means the model file is not exist or the password is not correct
|
|
}
|
|
|
|
_vehicleLPModel = CreateFilePath(_modelFolder, "vehiclelp.xml");
|
|
_vehicleAtModel = CreateFilePath(_modelFolder, "vehicle.xml");
|
|
_lprModel = CreateFilePath(_modelFolder, "lpr.xml");
|
|
|
|
ov::Core core;
|
|
int FLAGS_nthreads = 0;
|
|
|
|
std::set<std::string> devices;
|
|
for (const std::string& netDevices : { "GPU", "GPU", "GPU" }) {
|
|
if (netDevices.empty()) {
|
|
continue;
|
|
}
|
|
for (const std::string& device : parseDevices(netDevices)) {
|
|
devices.insert(device);
|
|
}
|
|
}
|
|
|
|
std::map<std::string, int32_t> device_nstreams = parseValuePerDevice(devices, "");
|
|
|
|
for (const std::string& device : devices) {
|
|
if ("CPU" == device) {
|
|
if (FLAGS_nthreads != 0) {
|
|
core.set_property("CPU", ov::inference_num_threads(FLAGS_nthreads));
|
|
}
|
|
//core.set_property("CPU", ov::affinity(ov::Affinity::NONE));
|
|
core.set_property("CPU", ov::streams::num((device_nstreams.count("CPU") > 0 ? ov::streams::Num(device_nstreams["CPU"]) : ov::streams::AUTO)));
|
|
|
|
device_nstreams["CPU"] = core.get_property("CPU", ov::streams::num);
|
|
}
|
|
if ("GPU" == device) {
|
|
core.set_property("GPU", ov::streams::num(device_nstreams.count("GPU") > 0 ? ov::streams::Num(device_nstreams["GPU"]) : ov::streams::AUTO));
|
|
|
|
device_nstreams["GPU"] = core.get_property("GPU", ov::streams::num);
|
|
if (devices.end() != devices.find("CPU")) {
|
|
core.set_property("GPU", ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel(1)));
|
|
}
|
|
}
|
|
}
|
|
double FLAGS_t = 0.5;
|
|
if(FileExist(_vehicleLPModel))_detector = new Detector(core, "GPU", _vehicleLPModel, { static_cast<float>(FLAGS_t), static_cast<float>(FLAGS_t) }, false);
|
|
else {
|
|
this->_logger->LogFatal("ANSALPR_OV::Initialize", _vehicleLPModel, __FILE__, __LINE__);
|
|
|
|
}
|
|
if(FileExist(_vehicleAtModel))_vehicleAttributesClassifier = new VehicleAttributesClassifier(core, "GPU", _vehicleAtModel, false);
|
|
else {
|
|
this->_logger->LogFatal("ANSALPR_OV::Initialize", _vehicleAtModel, __FILE__, __LINE__);
|
|
|
|
}
|
|
if(FileExist(_lprModel)) _lpr = new Lpr(core, "CPU", _lprModel, false);
|
|
else {
|
|
this->_logger->LogFatal("ANSALPR_OV::Initialize", _lprModel, __FILE__, __LINE__);
|
|
|
|
}
|
|
|
|
if (FileExist(_vehicleLPModel) &&
|
|
FileExist(_vehicleAtModel) &&
|
|
FileExist(_lprModel))return true;
|
|
else return false;
|
|
}
|
|
catch (std::exception& e) {
|
|
this->_logger->LogFatal("ANSALPR_OV::Initialize", e.what(), __FILE__, __LINE__);
|
|
return false;
|
|
}
|
|
};
|
|
bool ANSALPR_OV::Inference(const cv::Mat& input, std::string& lprResult) {
|
|
cv::Mat frame = input.clone();
|
|
std::shared_ptr<IInputSource> inputSource = std::make_shared<ImageSource>(frame, true);
|
|
std::vector<std::shared_ptr<InputChannel>> _inputChannels;
|
|
_inputChannels.push_back(InputChannel::create(inputSource));
|
|
unsigned nireq = 1;
|
|
bool isVideo = false;
|
|
std::size_t nclassifiersireq{ 0 };
|
|
std::size_t nrecognizersireq{ 0 };
|
|
nclassifiersireq = nireq * 3;
|
|
nrecognizersireq = nireq * 3;
|
|
|
|
Context context = { _inputChannels,
|
|
*_detector,
|
|
*_vehicleAttributesClassifier,
|
|
*_lpr,
|
|
2,
|
|
nireq,
|
|
isVideo,
|
|
nclassifiersireq,
|
|
nrecognizersireq };
|
|
|
|
std::shared_ptr<Worker> worker = std::make_shared<Worker>(2);
|
|
context.readersContext.readersWorker = worker;
|
|
context.inferTasksContext.inferTasksWorker = worker;
|
|
context.detectionsProcessorsContext.detectionsProcessorsWorker = worker;
|
|
context.resAggregatorsWorker = worker;
|
|
for (unsigned sourceID = 0; sourceID < _inputChannels.size(); sourceID++) {
|
|
VideoFrame::Ptr sharedVideoFrame = std::make_shared<ReborningVideoFrame>(context, sourceID, 0);
|
|
worker->push(std::make_shared<Reader>(sharedVideoFrame));
|
|
}
|
|
|
|
// Running
|
|
worker->runThreads();
|
|
worker->threadFunc();
|
|
worker->join();
|
|
|
|
std::list<BboxAndDescr> boxesAndDescrs = context.boxesAndDescrs;
|
|
std::vector<ALPRObject> output;
|
|
output.clear();
|
|
for(const BboxAndDescr boxesAndDescr: boxesAndDescrs)
|
|
{
|
|
if (boxesAndDescr.objectType == ANSCENTER::BboxAndDescr::ObjectType::PLATE) {
|
|
ALPRObject result;
|
|
result.classId = 0;
|
|
result.className = boxesAndDescr.descr;
|
|
result.confidence = 1.0;
|
|
result.box = boxesAndDescr.rect;
|
|
output.push_back(result);
|
|
}
|
|
}
|
|
lprResult = VectorDetectionToJsonString(output);
|
|
return true;
|
|
};
|
|
bool ANSALPR_OV::Inference(const cv::Mat& input, const std::vector<cv::Rect>& Bbox, std::string& lprResult) {
|
|
return true;
|
|
}
|
|
std::string ANSALPR_OV::VectorDetectionToJsonString(const std::vector<ALPRObject>& dets) {
|
|
boost::property_tree::ptree root;
|
|
boost::property_tree::ptree detectedObjects;
|
|
for (int i = 0; i < dets.size(); i++) {
|
|
boost::property_tree::ptree detectedNode;
|
|
detectedNode.put("class_id", dets[i].classId);
|
|
detectedNode.put("class_name", dets[i].className);
|
|
detectedNode.put("prob", dets[i].confidence);
|
|
detectedNode.put("x", dets[i].box.x);
|
|
detectedNode.put("y", dets[i].box.y);
|
|
detectedNode.put("width", dets[i].box.width);
|
|
detectedNode.put("height", dets[i].box.height);
|
|
detectedNode.put("mask", "");//Todo: convert masks to mask with comma seperated dets[i].mask);
|
|
detectedNode.put("extra_info", "");
|
|
|
|
// we might add masks into this using comma seperated string
|
|
detectedObjects.push_back(std::make_pair("", detectedNode));
|
|
}
|
|
root.add_child("results", detectedObjects);
|
|
std::ostringstream stream;
|
|
boost::property_tree::write_json(stream, root, false);
|
|
std::string trackingResult = stream.str();
|
|
return trackingResult;
|
|
}
|
|
} |