#include "ANSLPR_OD.h" #include "ANSTENSORRTOD.h" #include "ANSYOLOOD.h" #include "ANSTENSORRTCL.h" #include "ANSOPENVINOCL.h" #include "ANSRTYOLO.h" #include "ANSONNXYOLO.h" #include "ANSGpuFrameRegistry.h" #include #include #include #include #include #include // --------------------------------------------------------------------------- // Check ONNX model opset version by reading the protobuf header directly. // No dependency on onnx_pb.h / libprotobuf — just reads the raw bytes. // Returns 0 on failure. TRT 10.x supports up to opset ~17. // --------------------------------------------------------------------------- /// Read the default-domain opset version from an ONNX protobuf file. /// Returns 0 on failure (treat as "unknown — try TRT"). /// /// ONNX protobuf layout (ModelProto): /// field 8 (opset_import) = repeated OperatorSetIdProto { /// field 1 (domain) = string (empty for default domain) /// field 2 (version) = int64 /// } /// /// We scan the last 4 KB of the file because ONNX protobuf places /// opset_import (field 8) AFTER the graph (field 7), which can be /// hundreds of megabytes. The opset_import entry is always a tiny /// submessage (2-20 bytes) near the end of the file. /// Falls back to scanning the first 8 KB for older ONNX formats. static int GetOnnxOpsetVersion(const std::string& onnxPath) { std::ifstream f(onnxPath, std::ios::binary | std::ios::ate); if (!f.good()) return 0; auto fileSize = f.tellg(); if (fileSize < 16) return 0; // Helper lambda: scan a buffer for opset_import submessages auto scanForOpset = [](const unsigned char* buf, int bytesRead) -> int { int maxDefaultOpset = 0; for (int i = 0; i < bytesRead - 2; ++i) { if (buf[i] != 0x42) continue; int subLen = 0, lenBytes = 0; for (int b = i + 1; b < bytesRead && b < i + 4; ++b) { subLen |= (buf[b] & 0x7F) << (7 * lenBytes); lenBytes++; if ((buf[b] & 0x80) == 0) break; } if (subLen < 2 || subLen > 60) continue; int subStart = i + 1 + lenBytes; int subEnd = subStart + subLen; if (subEnd > bytesRead) continue; bool hasNonEmptyDomain = false; int version = 0; int pos = subStart; while (pos < subEnd) { unsigned char tag = buf[pos++]; int fieldNum = tag >> 3; int wireType = tag & 0x07; if (fieldNum == 1 && wireType == 2) { if (pos >= subEnd) break; int strLen = buf[pos++]; if (strLen > 0) hasNonEmptyDomain = true; pos += strLen; } else if (fieldNum == 2 && wireType == 0) { version = 0; int shift = 0; while (pos < subEnd) { unsigned char vb = buf[pos++]; version |= (vb & 0x7F) << shift; shift += 7; if ((vb & 0x80) == 0) break; } } else { break; } } if (!hasNonEmptyDomain && version > maxDefaultOpset) maxDefaultOpset = version; } return maxDefaultOpset; }; // Scan TAIL of file first (where opset_import usually lives) constexpr int TAIL_SIZE = 4096; std::streampos tailOffset = 0; if (fileSize > TAIL_SIZE) tailOffset = fileSize - static_cast(TAIL_SIZE); f.seekg(tailOffset, std::ios::beg); unsigned char buf[8192]; f.read(reinterpret_cast(buf), TAIL_SIZE); int bytesRead = static_cast(f.gcount()); int result = scanForOpset(buf, bytesRead); if (result > 0) return result; // Fallback: scan HEAD of file (older ONNX formats) f.seekg(0, std::ios::beg); f.read(reinterpret_cast(buf), 8192); bytesRead = static_cast(f.gcount()); return scanForOpset(buf, bytesRead); } // Write a message to Windows Event Log (Application log, source "ANSLogger"). // Visible in Event Viewer even when no console is attached (e.g. LabVIEW). static void WriteEventLog(const char* message, WORD eventType = EVENTLOG_INFORMATION_TYPE) { static HANDLE hLog = RegisterEventSourceA(NULL, "ANSLogger"); if (hLog) { const char* msgs[1] = { message }; ReportEventA(hLog, eventType, 0, 0, NULL, 1, 0, msgs, NULL); } } // --------------------------------------------------------------------------- // SEH wrapper for pre-building a single TRT engine from ONNX. // MSVC forbids __try in functions that use C++ object unwinding, // so the inner C++ logic lives in PreBuildOneModel_Impl (with try/catch) // and the outer SEH lives in PreBuildOneModel_SEH (no C++ objects). // --------------------------------------------------------------------------- struct PreBuildParams { const std::string* licenseKey; const std::string* modelFolder; std::string modelName; std::string className; std::string label; ANSCENTER::ModelConfig config; }; static bool PreBuildOneModel_Impl(const PreBuildParams& p) { try { auto tempDetector = std::make_unique(); tempDetector->SetLoadEngineOnCreation(false); ANSCENTER::ModelConfig cfg = p.config; cfg.modelType = ANSCENTER::ModelType::RTYOLO; std::string tempLabels; bool configured = tempDetector->LoadModelFromFolder( *p.licenseKey, cfg, p.modelName, p.className, *p.modelFolder, tempLabels); if (!configured) return false; // Try FP16 first std::string optimizedFolder; bool built = tempDetector->OptimizeModel(true /*fp16*/, optimizedFolder); // FP16 failed — retry with FP32 // Some ONNX models (especially opset 19+) crash TRT during FP16 tactic selection. if (!built) { std::cout << "[ANSALPR] Pre-build: FP16 failed for " << p.label << ", retrying with FP32..." << std::endl; // Recreate detector to get a clean Engine with FP32 precision tempDetector.reset(); tempDetector = std::make_unique(); tempDetector->SetLoadEngineOnCreation(false); tempLabels.clear(); configured = tempDetector->LoadModelFromFolder( *p.licenseKey, cfg, p.modelName, p.className, *p.modelFolder, tempLabels); if (configured) { built = tempDetector->OptimizeModel(false /*fp32*/, optimizedFolder); if (built) { std::cout << "[ANSALPR] Pre-build: " << p.label << " FP32 fallback succeeded." << std::endl; } } } tempDetector.reset(); // free VRAM return built; } catch (...) { return false; } } // FP32-only build — used as fallback when FP16 SEH-crashes static bool PreBuildOneModel_FP32Only(const PreBuildParams& p) { try { auto tempDetector = std::make_unique(); tempDetector->SetLoadEngineOnCreation(false); ANSCENTER::ModelConfig cfg = p.config; cfg.modelType = ANSCENTER::ModelType::RTYOLO; std::string tempLabels; bool configured = tempDetector->LoadModelFromFolder( *p.licenseKey, cfg, p.modelName, p.className, *p.modelFolder, tempLabels); if (!configured) return false; std::string optimizedFolder; bool built = tempDetector->OptimizeModel(false /*fp32*/, optimizedFolder); tempDetector.reset(); return built; } catch (...) { return false; } } static bool PreBuildOneModel_FP32Only_SEH(const PreBuildParams& p, DWORD* outCode) { *outCode = 0; __try { return PreBuildOneModel_FP32Only(p); } __except (EXCEPTION_EXECUTE_HANDLER) { *outCode = GetExceptionCode(); return false; } } // Pure SEH wrapper — no C++ objects, no try/catch // If FP16 SEH-crashes, automatically retries with FP32. static bool PreBuildOneModel_SEH(const PreBuildParams& p, DWORD* outCode) { *outCode = 0; __try { return PreBuildOneModel_Impl(p); } __except (EXCEPTION_EXECUTE_HANDLER) { *outCode = GetExceptionCode(); } // FP16 crashed — try FP32 fallback if (*outCode != 0) { std::cout << "[ANSALPR] Pre-build: " << p.label << " FP16 SEH crash (0x" << std::hex << *outCode << std::dec << "), retrying with FP32..." << std::endl; DWORD fp32Code = 0; bool fp32Ok = PreBuildOneModel_FP32Only_SEH(p, &fp32Code); if (fp32Ok) { std::cout << "[ANSALPR] Pre-build: " << p.label << " FP32 fallback succeeded." << std::endl; *outCode = 0; // clear error — FP32 worked return true; } // FP32 also failed — restore original error code if (fp32Code != 0) *outCode = fp32Code; } return false; } // --------------------------------------------------------------------------- // SEH wrapper for loading the LPC colour model (Step 5). // --------------------------------------------------------------------------- struct LoadLpcParams { const std::string* licenseKey; ANSCENTER::ModelConfig* config; const std::string* modelFolder; std::string* labels; std::unique_ptr* detector; }; static bool LoadLpcModel_Impl(const LoadLpcParams& p) { try { p.config->detectionType = ANSCENTER::DetectionType::CLASSIFICATION; p.config->modelType = ANSCENTER::ModelType::RTYOLO; auto rtyolo = std::make_unique(); bool ok = rtyolo->LoadModelFromFolder( *p.licenseKey, *p.config, "lpc", "lpc.names", *p.modelFolder, *p.labels); if (!ok) { return false; } rtyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, false); *p.detector = std::move(rtyolo); // upcast ANSRTYOLO -> ANSODBase return true; } catch (...) { p.detector->reset(); return false; } } static bool LoadLpcModel_SEH(const LoadLpcParams& p, DWORD* outCode) { *outCode = 0; __try { return LoadLpcModel_Impl(p); } __except (EXCEPTION_EXECUTE_HANDLER) { *outCode = GetExceptionCode(); return false; } } //#define FNS_DEBUG namespace ANSCENTER { // ---- Tunable constants for license plate recognition ---- constexpr float ROW_SPLIT_MIN_GAP_FACTOR = 0.2f; // maxGap < avgHeight * this => single row constexpr float ROW_SPLIT_AVGY_FACTOR = 0.4f; // avgY diff must exceed avgHeight * this constexpr size_t ROW_SPLIT_MIN_GROUP_SIZE = 2; // minimum chars per row constexpr float DUPLICATE_DIST_THRESHOLD = 5.0f; // pixels: chars closer than this are duplicates constexpr int DUPLICATE_GRID_SIZE = 10; // spatial hash grid cell size in pixels constexpr int ASYNC_TIMEOUT_SECONDS = 30; // timeout for async worker threads ANSALPR_OD::ANSALPR_OD() { valid = false; }; ANSALPR_OD::~ANSALPR_OD() { try { Destroy(); } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::~ANSALPR_OD", e.what(), __FILE__, __LINE__); } }; bool ANSALPR_OD::Destroy() { std::lock_guard lock(_mutex); try { if (this->_ocrDetector) this->_ocrDetector.reset(); if (this->_lpDetector) this->_lpDetector.reset(); if (this->_lpColourDetector) this->_lpColourDetector.reset(); return true; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::Destroy", e.what(), __FILE__, __LINE__); return false; } }; bool ANSALPR_OD::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword, double detectorThreshold, double ocrThreshold, double colourThreshold) { std::lock_guard lock(_mutex); try { _licenseKey = licenseKey; _licenseValid = false; _detectorThreshold = detectorThreshold; _ocrThreshold = ocrThreshold; _colorThreshold = colourThreshold; _country = Country::VIETNAM; CheckLicense(); if (!_licenseValid) { this->_logger.LogError("ANSALPR_OD::Initialize.", "License is not valid.", __FILE__, __LINE__); return false; } // Extract model folder // 0. Check if the modelZipFilePath exist? if (!FileExist(modelZipFilePath)) { this->_logger.LogFatal("ANSALPR_OD::Initialize", "Model zip file is not exist", __FILE__, __LINE__); } else { this->_logger.LogInfo("ANSALPR_OD::Initialize. Model zip file found: ", modelZipFilePath, __FILE__, __LINE__); } // 1. Unzip model zip file to a special location with folder name as model file (and version) std::string outputFolder; std::vector passwordArray; if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword); passwordArray.push_back("AnsDemoModels20@!"); passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag=="); passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg=="); std::string modelName = GetFileNameWithoutExtension(modelZipFilePath); size_t vectorSize = passwordArray.size(); for (size_t i = 0; i < vectorSize; i++) { if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false)) break; // Break the loop when the condition is met. } // 2. Check if the outputFolder exist if (!FolderExist(_modelFolder)) { this->_logger.LogError("ANSALPR_OD::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__); return false; // That means the model file is not exist or the password is not correct } // Check country std::string countryFile = CreateFilePath(_modelFolder, "country.txt"); if (FileExist(countryFile)) { std::ifstream infile(countryFile); std::string countryStr; std::getline(infile, countryStr); infile.close(); if (countryStr == "0") { _country = Country::VIETNAM; _plateFormats.push_back("ddlddddd"); _plateFormats.push_back("ddldddd"); _plateFormats.push_back("ddldddddd"); _plateFormats.push_back("ddllddddd"); _plateFormats.push_back("ddllddddd"); _plateFormats.push_back("ddMDdddddd"); _plateFormats.push_back("dddddNGdd"); _plateFormats.push_back("dddddQTdd"); _plateFormats.push_back("dddddCVdd"); _plateFormats.push_back("dddddNNdd"); _plateFormats.push_back("lldddd"); } else if (countryStr == "1") _country = Country::CHINA; else if (countryStr == "2") _country = Country::AUSTRALIA; else if (countryStr == "3") _country = Country::USA; else if (countryStr == "4") _country = Country::INDONESIA; else { _country = Country::VIETNAM;// Default _plateFormats.push_back("ddlddddd"); _plateFormats.push_back("ddldddd"); _plateFormats.push_back("ddldddddd"); _plateFormats.push_back("ddllddddd"); _plateFormats.push_back("ddllddddd"); _plateFormats.push_back("ddMDdddddd"); _plateFormats.push_back("dddddNGdd"); _plateFormats.push_back("dddddQTdd"); _plateFormats.push_back("dddddCVdd"); _plateFormats.push_back("dddddNNdd"); _plateFormats.push_back("lldddd"); } } else { _country = Country::VIETNAM;// Default _plateFormats.push_back("ddlddddd"); _plateFormats.push_back("ddldddd"); _plateFormats.push_back("ddldddddd"); _plateFormats.push_back("ddllddddd"); _plateFormats.push_back("ddllddddd"); _plateFormats.push_back("ddMDdddddd"); _plateFormats.push_back("dddddNGdd"); _plateFormats.push_back("dddddQTdd"); _plateFormats.push_back("dddddCVdd"); _plateFormats.push_back("dddddNNdd"); _plateFormats.push_back("lldddd"); } // 3. Load LD and OCR models alprChecker.Init(MAX_ALPR_FRAME); _lpColourModelConfig.detectionScoreThreshold = _colorThreshold; _lpdmodelConfig.detectionScoreThreshold= _detectorThreshold; _ocrModelConfig.detectionScoreThreshold= _ocrThreshold; return true; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::Initialize", e.what(), __FILE__, __LINE__); return false; } } bool ANSALPR_OD::LoadEngine() { std::lock_guard lock(_mutex); try { WriteEventLog("ANSALPR_OD::LoadEngine: Step 1 - Starting engine load"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 1: Starting engine load", __FILE__, __LINE__); // Check the hardware type _lpdmodelConfig.detectionScoreThreshold = _detectorThreshold; _ocrModelConfig.detectionScoreThreshold = _ocrThreshold; _lpColourModelConfig.detectionScoreThreshold = _colorThreshold; if (_lpdmodelConfig.detectionScoreThreshold < 0.25)_lpdmodelConfig.detectionScoreThreshold = 0.25; if (_ocrModelConfig.detectionScoreThreshold < 0.25)_ocrModelConfig.detectionScoreThreshold = 0.25; _lpdmodelConfig.modelConfThreshold = 0.5; _lpdmodelConfig.modelMNSThreshold = 0.5; _ocrModelConfig.modelConfThreshold = 0.5; _ocrModelConfig.modelMNSThreshold = 0.5; _lpColourModelConfig.modelConfThreshold = 0.5; _lpColourModelConfig.modelMNSThreshold = 0.5; _lpdmodelConfig.inpHeight = 640; _lpdmodelConfig.inpWidth = 640; _ocrModelConfig.inpHeight = 640; _ocrModelConfig.inpWidth = 640; _ocrModelConfig.gpuOptBatchSize = 8; _ocrModelConfig.gpuMaxBatchSize = 32; // desired max; engine builder auto-caps by GPU VRAM _ocrModelConfig.maxInputHeight = 640; _ocrModelConfig.maxInputWidth = 640; _ocrModelConfig.minInputHeight = 640; _ocrModelConfig.minInputWidth = 640; _ocrModelConfig.optInputHeight = 640; _ocrModelConfig.optInputWidth = 640; _lpColourModelConfig.inpHeight = 224; _lpColourModelConfig.inpWidth = 224; _lpColourModelConfig.gpuOptBatchSize = 8; _lpColourModelConfig.gpuMaxBatchSize = 32; // desired max; engine builder auto-caps by GPU VRAM _lpColourModelConfig.maxInputHeight = 224; _lpColourModelConfig.maxInputWidth = 224; _lpColourModelConfig.minInputHeight = 224; _lpColourModelConfig.minInputWidth = 224; _lpColourModelConfig.optInputHeight = 224; _lpColourModelConfig.optInputWidth = 224; std::string lprModel = CreateFilePath(_modelFolder, "lpd.onnx"); std::string lprClassesFile = CreateFilePath(_modelFolder, "lpd.names"); std::string ocrModel = CreateFilePath(_modelFolder, "ocr.onnx"); std::string ocrClassesFile = CreateFilePath(_modelFolder, "ocr.names"); std::string colorModel = CreateFilePath(_modelFolder, "lpc.xml"); std::string colorClassesFile = CreateFilePath(_modelFolder, "lpc.names"); WriteEventLog("ANSALPR_OD::LoadEngine: Step 2 - Checking hardware information"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 2: Checking hardware information", __FILE__, __LINE__); engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation();// WriteEventLog(("ANSALPR_OD::LoadEngine: Step 2 complete - Engine type = " + std::to_string(static_cast(engineType))).c_str()); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 2 complete: Engine type = " + std::to_string(static_cast(engineType)), __FILE__, __LINE__); valid = false; if (_lpDetector) _lpDetector.reset(); if (_ocrDetector) _ocrDetector.reset(); if (_lpColourDetector) _lpColourDetector.reset(); // ================================================================ // PRE-BUILD PASS: Build all TRT engine files before loading any. // // ANSALPR loads 3 models sequentially (LPD, OCR, LPC). When cached // .engine files exist, each one is just deserialized (low VRAM). // But after a driver/TRT update the caches are invalidated and every // model must be built from ONNX — a process that requires 2-5x the // final model size in temporary GPU workspace. // // Problem: if model #1 is built AND kept loaded, its VRAM footprint // reduces the workspace available for building model #2, which can // cause OOM crashes (LabVIEW error 1097) or very long hangs. // // Solution: use ANSRTYOLO::OptimizeModel() to build each .engine // file in a throwaway instance (buildWithRetry only — no load). // The instance is destroyed after saving, guaranteeing each build // gets the full GPU VRAM. The subsequent loading pass finds the // cached files and only needs the smaller deserialization memory. // ================================================================ if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { if (FileExist(lprModel) && FileExist(ocrModel)) { // Collect ONNX models that need TRT engine builds. // Each entry: { modelConfig, onnxModelName, classFileName, label } struct PreBuildSpec { ModelConfig config; std::string modelName; // e.g. "lpd" std::string className; // e.g. "lpd.names" std::string label; // for logging }; std::vector specs; { ModelConfig lpdCfg = _lpdmodelConfig; lpdCfg.detectionType = DetectionType::DETECTION; specs.push_back({ lpdCfg, "lpd", "lpd.names", "LPD" }); } { ModelConfig ocrCfg = _ocrModelConfig; ocrCfg.detectionType = DetectionType::DETECTION; specs.push_back({ ocrCfg, "ocr", "ocr.names", "OCR" }); } // LPC is optional and may be .xml (OpenVINO), only include if ONNX exists. // IMPORTANT: TRT 10.x crashes on opset 19+ ONNX models (access violation // in the ONNX parser that corrupts the CUDA context). Skip TRT pre-build // for high-opset models — they will fall through to ONNX Runtime at Step 5. std::string lpcOnnx = CreateFilePath(_modelFolder, "lpc.onnx"); bool lpcSkipTrt = false; if (FileExist(lpcOnnx) && (_lpColourModelConfig.detectionScoreThreshold > 0)) { int lpcOpset = GetOnnxOpsetVersion(lpcOnnx); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build: LPC ONNX opset detected = " + std::to_string(lpcOpset), __FILE__, __LINE__); if (lpcOpset > 17 || lpcOpset == 0) { // opset > 17: TRT crashes on these models // opset == 0: detection failed, assume high opset (safer than crashing) lpcSkipTrt = true; this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build: LPC opset " + std::to_string(lpcOpset) + " > 17, skipping TRT (will use ONNX Runtime instead)", __FILE__, __LINE__); WriteEventLog(("ANSALPR_OD::LoadEngine: LPC opset " + std::to_string(lpcOpset) + " too high for TRT, using ORT").c_str()); } else { ModelConfig lpcCfg = _lpColourModelConfig; lpcCfg.detectionType = DetectionType::CLASSIFICATION; specs.push_back({ lpcCfg, "lpc", "lpc.names", "LPC" }); } } // Quick check: do ANY engines need building? // If all are cached, skip entirely for zero overhead on normal launches. // // IMPORTANT: Apply the same GPU-tier batch cap that buildLoadNetwork() // applies internally. Without this, the probe looks for e.g. // "ocr.engine...b32" but the actual build saved "ocr.engine...b16" // (capped by VRAM), causing needless rebuilds every launch. int gpuMaxBatch = 1; { auto gpus = Engine::enumerateDevices(); if (!gpus.empty()) { const size_t totalMiB = gpus[0].totalMemoryBytes / (1024ULL * 1024); if (totalMiB >= 15800) gpuMaxBatch = 32; // ~16 GiB+ else if (totalMiB >= 11800) gpuMaxBatch = 16; // ~12 GiB else if (totalMiB >= 7900) gpuMaxBatch = 8; // ~ 8 GiB (batch=16 OCR exec ctx ~987 MiB, too large for 4 concurrent tasks) else if (totalMiB >= 3900) gpuMaxBatch = 4; // ~ 4 GiB else if (totalMiB >= 1900) gpuMaxBatch = 2; // ~ 2 GiB else gpuMaxBatch = 1; } } bool anyNeedsBuild = false; for (auto& spec : specs) { ANSCENTER::Options o; o.optBatchSize = spec.config.gpuOptBatchSize; o.maxBatchSize = spec.config.gpuMaxBatchSize; o.deviceIndex = spec.config.gpuDeviceIndex; o.maxInputHeight = spec.config.maxInputHeight; o.minInputHeight = spec.config.minInputHeight; o.optInputHeight = spec.config.optInputHeight; o.maxInputWidth = spec.config.maxInputWidth; o.minInputWidth = spec.config.minInputWidth; o.optInputWidth = spec.config.optInputWidth; o.engineFileDir = _modelFolder; o.precision = ANSCENTER::Precision::FP16; // Apply GPU-tier batch cap (must match buildLoadNetwork behavior) if (o.maxBatchSize > gpuMaxBatch) { o.maxBatchSize = gpuMaxBatch; o.optBatchSize = std::min(o.optBatchSize, o.maxBatchSize); } auto probe = std::make_unique>(o); std::string fp16File = probe->serializeEngineOptions(o, CreateFilePath(_modelFolder, spec.modelName + ".onnx")); o.precision = ANSCENTER::Precision::FP32; std::string fp32File = probe->serializeEngineOptions(o, CreateFilePath(_modelFolder, spec.modelName + ".onnx")); probe.reset(); if (!FileExist(fp16File) && !FileExist(fp32File)) { anyNeedsBuild = true; this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build: " + spec.label + " engine not cached, build required", __FILE__, __LINE__); } } if (anyNeedsBuild) { WriteEventLog("ANSALPR_OD::LoadEngine: Pre-build pass starting - optimizing engines one-at-a-time"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build pass: optimizing engine files one-at-a-time with full GPU VRAM", __FILE__, __LINE__); for (auto& spec : specs) { this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build: Optimizing " + spec.label + " engine...", __FILE__, __LINE__); WriteEventLog(("ANSALPR_OD::LoadEngine: Pre-build: Optimizing " + spec.label + " engine...").c_str()); PreBuildParams pbp; pbp.licenseKey = &_licenseKey; pbp.modelFolder = &_modelFolder; pbp.modelName = spec.modelName; pbp.className = spec.className; pbp.label = spec.label; pbp.config = spec.config; DWORD sehCode = 0; bool built = PreBuildOneModel_SEH(pbp, &sehCode); if (sehCode != 0) { char buf[256]; snprintf(buf, sizeof(buf), "ANSALPR_OD::LoadEngine: Pre-build: %s SEH exception 0x%08X - skipping", spec.label.c_str(), sehCode); WriteEventLog(buf, EVENTLOG_ERROR_TYPE); this->_logger.LogError("ANSALPR_OD::LoadEngine", "Pre-build: " + spec.label + " SEH crash, skipping", __FILE__, __LINE__); } else if (built) { this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build: " + spec.label + " engine built and cached successfully", __FILE__, __LINE__); WriteEventLog(("ANSALPR_OD::LoadEngine: Pre-build: " + spec.label + " engine built OK").c_str()); } else { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Pre-build: " + spec.label + " engine build failed (will retry in load pass)", __FILE__, __LINE__); WriteEventLog(("ANSALPR_OD::LoadEngine: Pre-build: " + spec.label + " build FAILED").c_str(), EVENTLOG_WARNING_TYPE); } } this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Pre-build pass complete. Proceeding to load all engines.", __FILE__, __LINE__); } } } if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { if (FileExist(lprModel) && (FileExist(ocrModel))) { WriteEventLog("ANSALPR_OD::LoadEngine: Step 3 - Loading LP detector with TensorRT"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 3: Loading LP detector with TensorRT", __FILE__, __LINE__); _lpdmodelConfig.detectionType = DetectionType::DETECTION; _lpdmodelConfig.modelType = ModelType::RTYOLO; _lpDetector = std::make_unique();// TensorRT bool lpSuccess = _lpDetector->LoadModelFromFolder(_licenseKey, _lpdmodelConfig, "lpd", "lpd.names", _modelFolder, _lpdLabels); if (!lpSuccess) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load LP detector (TensorRT). GPU may not support this model.", __FILE__, __LINE__); _lpDetector.reset(); } else { // Enable tracker on LP detector for stable bounding box tracking, // but disable stabilization (no ghost plates — ALPRChecker handles text stabilization) _lpDetector->SetTracker(TrackerType::BYTETRACK, true); _lpDetector->SetStabilization(false); } WriteEventLog("ANSALPR_OD::LoadEngine: Step 4 - Loading OCR detector with TensorRT"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 4: Loading OCR detector with TensorRT", __FILE__, __LINE__); _ocrModelConfig.detectionType = DetectionType::DETECTION; _ocrModelConfig.modelType = ModelType::RTYOLO; _ocrDetector = std::make_unique();// TensorRT bool ocrSuccess = _ocrDetector->LoadModelFromFolder(_licenseKey, _ocrModelConfig, "ocr", "ocr.names", _modelFolder, _ocrLabels); if (!ocrSuccess) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load OCR detector (TensorRT). GPU may not support this model.", __FILE__, __LINE__); _ocrDetector.reset(); } else { _ocrDetector->SetTracker(TrackerType::BYTETRACK, false); } // Check if we need to load the color model (optional — SEH-protected) if (FileExist(colorModel) && (_lpColourModelConfig.detectionScoreThreshold > 0)) { // Route decision: use ONNX Runtime for high-opset models that crash TRT int lpcOpsetCheck = GetOnnxOpsetVersion(CreateFilePath(_modelFolder, "lpc.onnx")); // opset > 17: TRT crashes on these models // opset == 0: detection failed, assume high opset (safer than crashing) bool lpcSkipTrt = (lpcOpsetCheck > 17 || lpcOpsetCheck == 0); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 5: LPC opset detected = " + std::to_string(lpcOpsetCheck) + ", skipTrt = " + (lpcSkipTrt ? std::string("true") : std::string("false")), __FILE__, __LINE__); if (lpcSkipTrt) { WriteEventLog("ANSALPR_OD::LoadEngine: Step 5 - Loading colour classifier with ONNX Runtime (opset > 17)"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 5: Loading colour classifier with ONNX Runtime (opset too high for TRT)", __FILE__, __LINE__); try { _lpColourModelConfig.detectionType = DetectionType::CLASSIFICATION; _lpColourModelConfig.modelType = ModelType::ONNXYOLO; auto ortDetector = std::make_unique(); bool ok = ortDetector->LoadModelFromFolder( _licenseKey, _lpColourModelConfig, "lpc", "lpc.names", _modelFolder, _lpColourLabels); if (ok) { ortDetector->SetTracker(TrackerType::BYTETRACK, false); _lpColourDetector = std::move(ortDetector); WriteEventLog("ANSALPR_OD::LoadEngine: Step 5 - LPC loaded via ONNX Runtime"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 5: Colour classifier loaded via ONNX Runtime", __FILE__, __LINE__); } else { WriteEventLog("ANSALPR_OD::LoadEngine: Step 5 - LPC ONNX Runtime load failed", EVENTLOG_ERROR_TYPE); this->_logger.LogError("ANSALPR_OD::LoadEngine", "Step 5: Colour classifier ONNX Runtime load failed. Colour detection disabled.", __FILE__, __LINE__); } } catch (const std::exception& e) { WriteEventLog(("ANSALPR_OD::LoadEngine: Step 5 - LPC exception: " + std::string(e.what())).c_str(), EVENTLOG_ERROR_TYPE); this->_logger.LogError("ANSALPR_OD::LoadEngine", "Step 5: Colour classifier exception: " + std::string(e.what()), __FILE__, __LINE__); _lpColourDetector.reset(); } catch (...) { WriteEventLog("ANSALPR_OD::LoadEngine: Step 5 - LPC unknown exception", EVENTLOG_ERROR_TYPE); this->_logger.LogError("ANSALPR_OD::LoadEngine", "Step 5: Colour classifier unknown exception. Colour detection disabled.", __FILE__, __LINE__); _lpColourDetector.reset(); } } else { // Normal TRT path (opset ≤ 17) WriteEventLog("ANSALPR_OD::LoadEngine: Step 5 - Loading colour classifier with TensorRT"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 5: Loading colour classifier with TensorRT", __FILE__, __LINE__); LoadLpcParams lpc; lpc.licenseKey = &_licenseKey; lpc.config = &_lpColourModelConfig; lpc.modelFolder = &_modelFolder; lpc.labels = &_lpColourLabels; lpc.detector = &_lpColourDetector; DWORD sehCode = 0; bool colourOk = LoadLpcModel_SEH(lpc, &sehCode); if (sehCode != 0) { char buf[256]; snprintf(buf, sizeof(buf), "ANSALPR_OD::LoadEngine: Step 5 LPC SEH exception 0x%08X - colour detection disabled", sehCode); WriteEventLog(buf, EVENTLOG_ERROR_TYPE); this->_logger.LogError("ANSALPR_OD::LoadEngine", "Step 5: Colour classifier crashed (SEH). Colour detection disabled.", __FILE__, __LINE__); _lpColourDetector.reset(); } else if (!colourOk) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load colour detector (TensorRT). Colour detection disabled.", __FILE__, __LINE__); } } } // TensorRT failed for both critical models — fall back to ONNX Runtime if (!lpSuccess || !ocrSuccess) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "TensorRT engine build failed. Falling back to ONNX Runtime...", __FILE__, __LINE__); if (_lpDetector) _lpDetector.reset(); if (_ocrDetector) _ocrDetector.reset(); if (_lpColourDetector) _lpColourDetector.reset(); // Fall through to ONNX path below engineType = ANSCENTER::EngineType::CPU; } else { valid = true; } } } // ONNX Runtime fallback path (CPU or when TensorRT fails) if (!valid) { if (FileExist(lprModel) && (FileExist(ocrModel))) { WriteEventLog("ANSALPR_OD::LoadEngine: Step 6 - Loading LP detector with ONNX Runtime"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 6: Loading LP detector with ONNX Runtime", __FILE__, __LINE__); _lpdmodelConfig.detectionType = DetectionType::DETECTION; _lpdmodelConfig.modelType = ModelType::ONNXYOLO; std::string _lprClasses; _lpDetector = std::make_unique();// Yolo bool lpSuccess = _lpDetector->LoadModelFromFolder(_licenseKey, _lpdmodelConfig, "lpd", "lpd.names", _modelFolder, _lprClasses); if (!lpSuccess) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__); _lpDetector.reset(); } else { // Enable tracker on LP detector for stable bounding box tracking, // but disable stabilization (no ghost plates — ALPRChecker handles text stabilization) _lpDetector->SetTracker(TrackerType::BYTETRACK, true); _lpDetector->SetStabilization(false); } WriteEventLog("ANSALPR_OD::LoadEngine: Step 7 - Loading OCR detector with ONNX Runtime"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 7: Loading OCR detector with ONNX Runtime", __FILE__, __LINE__); _ocrModelConfig.detectionType = DetectionType::DETECTION; _ocrModelConfig.modelType = ModelType::ONNXYOLO; _ocrDetector = std::make_unique();// Yolo bool ocrSuccess = _ocrDetector->LoadModelFromFolder(_licenseKey, _ocrModelConfig, "ocr", "ocr.names", _modelFolder, _ocrLabels); if (!ocrSuccess) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load OCR detector (ONNX Runtime).", __FILE__, __LINE__); _ocrDetector.reset(); } else { _ocrDetector->SetTracker(TrackerType::BYTETRACK, false); } // Check if we need to load the color model if (FileExist(colorModel) && (_lpColourModelConfig.detectionScoreThreshold > 0)) { _lpColourModelConfig.detectionType = DetectionType::CLASSIFICATION; _lpColourModelConfig.modelType = ModelType::ONNXYOLO; _lpColourDetector = std::make_unique();// Classification with ONNX bool colourSuccess = _lpColourDetector->LoadModelFromFolder(_licenseKey, _lpColourModelConfig, "lpc", "lpc.names", _modelFolder, _lpColourLabels); if (!colourSuccess) { this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__); _lpColourDetector.reset(); } else { _lpColourDetector->SetTracker(TrackerType::BYTETRACK, false); } } if (lpSuccess && ocrSuccess) { valid = true; if (engineType == ANSCENTER::EngineType::CPU) { this->_logger.LogDebug("ANSALPR_OD::LoadEngine", "Successfully loaded models with ONNX Runtime fallback.", __FILE__, __LINE__); } } else { this->_logger.LogFatal("ANSALPR_OD::LoadEngine", "Failed to load critical models with both TensorRT and ONNX Runtime.", __FILE__, __LINE__); } } } _isInitialized = valid; WriteEventLog(("ANSALPR_OD::LoadEngine: Step 8 - Engine load complete. Valid = " + std::to_string(valid)).c_str()); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 8: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__); return valid; } catch (std::exception& e) { WriteEventLog(("ANSALPR_OD::LoadEngine: C++ exception: " + std::string(e.what())).c_str(), EVENTLOG_ERROR_TYPE); this->_logger.LogFatal("ANSALPR_OD::LoadEngine", std::string("C++ exception: ") + e.what(), __FILE__, __LINE__); return false; } } bool ANSALPR_OD::shouldUseALPRChecker(const cv::Size& imageSize, const std::string& cameraId) { // Force disabled → never use if (!_enableALPRChecker) return false; // Small images are always pipeline crops — skip auto-detection if (imageSize.width < ImageSizeTracker::MIN_FULLFRAME_WIDTH) return false; // Enabled: auto-detect pipeline vs full-frame by exact image size consistency. // Full-frame: same resolution every frame (e.g., 3840x2160 always). // Pipeline crops: vary by a few pixels (e.g., 496x453, 497x455) — exact match fails. auto& tracker = _imageSizeTrackers[cameraId]; bool wasFullFrame = tracker.detectedFullFrame; if (imageSize == tracker.lastSize) { tracker.consistentCount++; if (tracker.consistentCount >= ImageSizeTracker::CONFIRM_THRESHOLD) { tracker.detectedFullFrame = true; } } else { tracker.lastSize = imageSize; tracker.consistentCount = 1; tracker.detectedFullFrame = false; } // Log state transitions if (tracker.detectedFullFrame != wasFullFrame) { ANS_DBG("ALPR_Checker", "cam=%s mode auto-detected: %s (img=%dx%d consistent=%d)", cameraId.c_str(), tracker.detectedFullFrame ? "FULL-FRAME (Layer2+3 ON)" : "PIPELINE (Layer2+3 OFF)", imageSize.width, imageSize.height, tracker.consistentCount); } return tracker.detectedFullFrame; } std::vector ANSALPR_OD::RunInferenceSingleFrame(const cv::Mat& input, const std::string& cameraId) { // No coarse _mutex here — sub-components (detectors, alprChecker) have their own locks. // LabVIEW semaphore controls concurrency at the caller level. // Early validation if (!_licenseValid) { this->_logger.LogError("ANSALPR_OD::RunInference", "Invalid license", __FILE__, __LINE__); return {}; } if (!valid) { this->_logger.LogError("ANSALPR_OD::RunInference", "Invalid model", __FILE__, __LINE__); return {}; } if (!_isInitialized) { this->_logger.LogError("ANSALPR_OD::RunInference", "Model is not initialized", __FILE__, __LINE__); return {}; } if (input.empty()) { this->_logger.LogError("ANSALPR_OD::RunInference", "Input image is empty", __FILE__, __LINE__); return {}; } if (input.cols < 5 || input.rows < 5) { this->_logger.LogError("ANSALPR_OD::RunInference", "Input image size is too small", __FILE__, __LINE__); return {}; } if (!this->_lpDetector) { this->_logger.LogFatal("ANSALPR_OD::RunInference", "_lprDetector is null", __FILE__, __LINE__); return {}; } if (!this->_ocrDetector) { this->_logger.LogFatal("ANSALPR_OD::RunInference", "PPOCR instance is null", __FILE__, __LINE__); return {}; } try { // Convert grayscale to BGR if necessary (use local buffer for thread safety) cv::Mat localFrame; if (input.channels() == 1) { cv::cvtColor(input, localFrame, cv::COLOR_GRAY2BGR); } const cv::Mat& frame = (input.channels() == 1) ? localFrame : input; const int frameWidth = frame.cols; const int frameHeight = frame.rows; #ifdef FNS_DEBUG cv::Mat draw = input.clone(); #endif // Use local variable instead of shared _detectedArea for thread safety cv::Rect detectedArea(0, 0, frameWidth, frameHeight); if (detectedArea.width <= 50 || detectedArea.height <= 50) { return {}; } #ifdef FNS_DEBUG cv::rectangle(draw, detectedArea, cv::Scalar(0, 0, 255), 2); #endif // Run license plate detection cv::Mat activeFrame = frame(detectedArea); std::vector lprOutput = _lpDetector->RunInference(activeFrame, cameraId); for (size_t _di = 0; _di < lprOutput.size(); ++_di) { ANS_DBG("ALPR_Track", "cam=%s det[%zu] tid=%d box=(%d,%d,%d,%d) conf=%.2f", cameraId.c_str(), _di, lprOutput[_di].trackId, lprOutput[_di].box.x, lprOutput[_di].box.y, lprOutput[_di].box.width, lprOutput[_di].box.height, lprOutput[_di].confidence); } if (lprOutput.empty()) { #ifdef FNS_DEBUG cv::resize(draw, draw, cv::Size(1920, 1080)); cv::imshow("Detected Areas", draw); cv::waitKey(1); #endif return {}; } std::vector output; output.reserve(lprOutput.size()); for (auto& lprObject : lprOutput) { const cv::Rect& box = lprObject.box; #ifdef FNS_DEBUG cv::rectangle(draw, box, cv::Scalar(0, 255, 255), 2); #endif // Calculate cropped region (padding = 0) const int x1 = std::max(0, box.x); const int y1 = std::max(0, box.y); const int width = std::min(frameWidth - x1, box.width); const int height = std::min(frameHeight - y1, box.height); if (width <= 0 || height <= 0) { continue; } cv::Rect lprPos(x1, y1, width, height); cv::Mat alignedLPR = frame(lprPos);// .clone(); // OCR inference std::string ocrText = DetectLicensePlateString(alignedLPR, cameraId); if (ocrText.empty()) { continue; } lprObject.cameraId = cameraId; if (shouldUseALPRChecker(cv::Size(frameWidth, frameHeight), cameraId)) { lprObject.className = alprChecker.checkPlateByTrackId(cameraId, ocrText, lprObject.trackId); } else { lprObject.className = ocrText; } if (lprObject.className.empty()) { continue; } std::string colour = DetectLPColourCached(alignedLPR, cameraId, lprObject.className); if (!colour.empty()) { lprObject.extraInfo = "color:" + colour; } output.push_back(std::move(lprObject)); } #ifdef FNS_DEBUG cv::resize(draw, draw, cv::Size(1920, 1080)); cv::imshow("Detected Areas", draw); cv::waitKey(1); #endif // Deduplicate: if two trackIds claim the same plate text, keep the one // with the higher accumulated score to prevent plate flickering if (shouldUseALPRChecker(cv::Size(frameWidth, frameHeight), cameraId)) { ensureUniquePlateText(output, cameraId); } return output; } catch (const cv::Exception& e) { this->_logger.LogFatal("ANSALPR_OD::RunInference", std::string("OpenCV Exception: ") + e.what(), __FILE__, __LINE__); } catch (const std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::RunInference", e.what(), __FILE__, __LINE__); } catch (...) { this->_logger.LogFatal("ANSALPR_OD::RunInference", "Unknown exception occurred", __FILE__, __LINE__); } return {}; } std::string ANSALPR_OD::DetectLicensePlateString(const cv::Mat& lprROI, const std::string& cameraId) { // No coarse _mutex — _ocrDetector has its own m_inferenceMutex try { // convert lprROI to greyscale if it is not already if (lprROI.empty()) { this->_logger.LogError("ANSALPR_OD::DetectLicensePlateString", "Input image is empty", __FILE__, __LINE__); return ""; } cv::Mat grayLprROI; if (lprROI.channels() == 3) { cv::cvtColor(lprROI, grayLprROI, cv::COLOR_BGR2GRAY); } else { grayLprROI = lprROI; } std::vector ocrOutput = _ocrDetector->RunInference(grayLprROI, cameraId); std::string ocrText = ""; if (ocrOutput.empty()) return ocrText; //std::cout << "=== OCR Detections ===" << std::endl; //for (size_t i = 0; i < ocrOutput.size(); ++i) { // << "' X=" << ocrOutput[i].box.x // << " Y=" << ocrOutput[i].box.y << std::endl; //} // Remove duplicates std::vector uniqueOutput; for (const auto& obj : ocrOutput) { bool isDuplicate = false; for (const auto& unique : uniqueOutput) { if (std::abs(obj.box.x - unique.box.x) < DUPLICATE_DIST_THRESHOLD && std::abs(obj.box.y - unique.box.y) < DUPLICATE_DIST_THRESHOLD) { isDuplicate = true; break; } } if (!isDuplicate) { uniqueOutput.push_back(obj); } } //std::cout << "\nAfter removing duplicates: " << uniqueOutput.size() << " chars" << std::endl; if (uniqueOutput.empty()) return ocrText; if (uniqueOutput.size() == 1) return uniqueOutput[0].className; // Calculate average character height for threshold float avgHeight = 0; for (const auto& obj : uniqueOutput) { avgHeight += obj.box.height; } avgHeight /= uniqueOutput.size(); // Calculate linear regression Y = mX + b float sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0; int n = uniqueOutput.size(); for (const auto& obj : uniqueOutput) { float x = obj.box.x; float y = obj.box.y; sumX += x; sumY += y; sumXY += x * y; sumX2 += x * x; } float denominator = n * sumX2 - sumX * sumX; float slope = (std::abs(denominator) > 1e-6f) ? (n * sumXY - sumX * sumY) / denominator : 0.0f; float intercept = (n > 0) ? (sumY - slope * sumX) / n : 0.0f; //std::cout << "Linear regression: Y = " << slope << " * X + " << intercept << std::endl; // Calculate perpendicular distance from each point to regression line float a = slope; float b = -1.0f; float c = intercept; float normFactor = std::sqrt(a * a + b * b); std::vector> distances; for (size_t i = 0; i < uniqueOutput.size(); ++i) { float x = uniqueOutput[i].box.x; float y = uniqueOutput[i].box.y; float dist = (a * x + b * y + c) / normFactor; distances.push_back({ dist, i }); } // Sort by perpendicular distance std::sort(distances.begin(), distances.end(), [](const std::pair& a, const std::pair& b) { return a.first < b.first; }); // Find largest gap in perpendicular distances float maxGap = 0; size_t splitIdx = distances.size() / 2; //std::cout << "\n=== Distance gaps ===" << std::endl; for (size_t i = 1; i < distances.size(); ++i) { float gap = distances[i].first - distances[i - 1].first; //std::cout << "Gap " << i << ": " << gap << std::endl; if (gap > maxGap) { maxGap = gap; splitIdx = i; } } // Check if this is actually a single row // If max gap is too small relative to character height, it's a single row std::vector> rows; if (maxGap < avgHeight * ROW_SPLIT_MIN_GAP_FACTOR) { // Single row - all characters on one line rows.resize(1); rows[0] = uniqueOutput; } else { // Two rows rows.resize(2); // Split into two rows based on perpendicular distance for (size_t i = 0; i < distances.size(); ++i) { size_t objIdx = distances[i].second; if (i < splitIdx) { rows[0].push_back(uniqueOutput[objIdx]); } else { rows[1].push_back(uniqueOutput[objIdx]); } } // Determine which row is on top (lower average Y = top row) float avgY0 = 0, avgY1 = 0; for (const auto& obj : rows[0]) { avgY0 += obj.box.y; } for (const auto& obj : rows[1]) { avgY1 += obj.box.y; } avgY0 /= rows[0].size(); avgY1 /= rows[1].size(); //std::cout << "Average Y: Row0=" << avgY0 << " Row1=" << avgY1 << std::endl; // Swap if needed (top row should be row 0) if (avgY0 > avgY1) { //std::cout << "Swapping rows (Row 0 should be on top)" << std::endl; std::swap(rows[0], rows[1]); } } //std::cout << "\n=== Rows before X-sorting ===" << std::endl; //for (size_t r = 0; r < rows.size(); ++r) { // std::cout << "Row " << r << " (" << rows[r].size() << " chars): "; // for (const auto& obj : rows[r]) { // } // std::cout << std::endl; //} // Sort each row by X for (auto& row : rows) { std::sort(row.begin(), row.end(), [](const Object& a, const Object& b) { return a.box.x < b.box.x; }); } /*std::cout << "\n=== Rows after X-sorting ===" << std::endl; for (size_t r = 0; r < rows.size(); ++r) { std::cout << "Row " << r << ": "; for (const auto& obj : rows[r]) { } std::cout << std::endl; }*/ // Concatenate for (const auto& row : rows) { for (const auto& obj : row) { ocrText += obj.className; } } //std::cout << "\nFinal text: '" << ocrText << "'" << std::endl; grayLprROI.release(); // Release the grayscale image to free memory std::string processedOcrText = AnalyseLicensePlateText(ocrText); return processedOcrText; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::DetectLicensePlateString", e.what(), __FILE__, __LINE__); return ""; } } std::string ANSALPR_OD::AnalyseLicensePlateText(const std::string& ocrText) { std::string analysedLP = ""; try { std::string cleanOCRText = ""; for (size_t i = 0; i < ocrText.size(); ++i) { char c = ocrText[i]; if (std::isalnum(c))cleanOCRText += c; } std::transform(cleanOCRText.begin(), cleanOCRText.end(), cleanOCRText.begin(), ::toupper); int ocrSize = cleanOCRText.size(); switch (_country) { case Country::VIETNAM: analysedLP = cleanOCRText; break; case Country::INDONESIA: analysedLP = cleanOCRText; break; case Country::AUSTRALIA: analysedLP = cleanOCRText; break; case Country::USA: break; } // Format validation: reject plates that don't match any configured format if (!analysedLP.empty() && !_plateFormats.empty() && !MatchesPlateFormat(analysedLP)) { return ""; } return analysedLP; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::AnalyseLicensePlateText", e.what(), __FILE__, __LINE__); return ""; } } bool ANSALPR_OD::MatchesPlateFormat(const std::string& plate) const { if (_plateFormats.empty()) { return true; // No formats configured - accept all } for (const auto& format : _plateFormats) { if (plate.size() != format.size()) continue; bool matches = true; for (size_t i = 0; i < format.size(); ++i) { char f = format[i]; char p = plate[i]; if (f == 'd') { if (!std::isdigit(static_cast(p))) { matches = false; break; } } else if (f == 'l') { if (!std::isalpha(static_cast(p))) { matches = false; break; } } else { if (p != f) { matches = false; break; } // Fixed letter (A-Z) or other literal } } if (matches) return true; } return false; } std::string ANSALPR_OD::DetectLPColourDetector(const cv::Mat& lprROI, const std::string& cameraId) { // Early validation - no lock needed for these checks if (_lpColourModelConfig.detectionScoreThreshold <= 0.0f) { return {}; // Colour detection not enabled } if (!_lpColourDetector) { return {}; } if (lprROI.empty()) { this->_logger.LogError("ANSALPR_OD::DetectLPColourDetector", "Input image is empty", __FILE__, __LINE__); return {}; } // No coarse _mutex — _lpColourDetector has its own m_inferenceMutex try { std::vector colourOutputs = _lpColourDetector->RunInference(lprROI, cameraId); if (colourOutputs.empty()) { return {}; } // Find detection with highest confidence const auto& bestDetection = *std::max_element( colourOutputs.begin(), colourOutputs.end(), [](const Object& a, const Object& b) { return a.confidence < b.confidence; } ); return bestDetection.className; } catch (const std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::DetectLPColourDetector", e.what(), __FILE__, __LINE__); return {}; } } std::string ANSALPR_OD::DetectLPColourCached(const cv::Mat& lprROI, const std::string& cameraId, const std::string& plateText) { // Empty plate text = can't cache, fall through to full inference if (plateText.empty()) { return DetectLPColourDetector(lprROI, cameraId); } // Check cache first (fine-grained lock, no GPU work) { std::lock_guard cacheLock(_colourCacheMutex); auto it = _colourCache.find(plateText); if (it != _colourCache.end()) { it->second.hitCount++; return it->second.colour; // Cache hit — 0ms } } // Cache miss — run the actual classifier (no lock held during GPU inference) std::string colour = DetectLPColourDetector(lprROI, cameraId); // Store in cache (fine-grained lock) if (!colour.empty()) { std::lock_guard cacheLock(_colourCacheMutex); if (_colourCache.size() >= COLOUR_CACHE_MAX_SIZE) { _colourCache.clear(); } _colourCache[plateText] = { colour, 0 }; } return colour; } bool ANSALPR_OD::Inference(const cv::Mat& input, std::string& lprResult) { // No coarse _mutex — delegates to Inference(input, lprResult, cameraId) which is also lock-free if (input.empty()) return false; if ((input.cols < 5) || (input.rows < 5)) return false; return Inference(input, lprResult, "CustomCam"); } bool ANSALPR_OD::Inference(const cv::Mat& input, std::string& lprResult, const std::string& cameraId) { // No coarse _mutex — sub-components have their own fine-grained locks. // LabVIEW semaphore controls concurrency at the caller level. // Early validation if (!_licenseValid) { this->_logger.LogError("ANSALPR_OD::Inference", "Invalid license", __FILE__, __LINE__); return false; } if (!valid) { this->_logger.LogError("ANSALPR_OD::Inference", "Invalid model", __FILE__, __LINE__); return false; } if (!_isInitialized) { this->_logger.LogError("ANSALPR_OD::Inference", "Model is not initialized", __FILE__, __LINE__); return false; } if (input.empty()) { this->_logger.LogError("ANSALPR_OD::Inference", "Input image is empty", __FILE__, __LINE__); return false; } if (input.cols < 5 || input.rows < 5) { this->_logger.LogError("ANSALPR_OD::Inference", "Input image size is too small", __FILE__, __LINE__); return false; } if (!this->_lpDetector) { this->_logger.LogFatal("ANSALPR_OD::Inference", "_lpDetector is null", __FILE__, __LINE__); return false; } std::vector output; try { // --- Debug timer helper (zero-cost when _debugFlag == false) --- using Clock = std::chrono::steady_clock; const bool dbg = _debugFlag; auto t0 = dbg ? Clock::now() : Clock::time_point{}; auto tPrev = t0; auto elapsed = [&]() -> double { auto now = Clock::now(); double ms = std::chrono::duration(now - tPrev).count(); tPrev = now; return ms; }; // Convert grayscale to BGR if necessary (use local buffer for thread safety) cv::Mat localFrame; if (input.channels() == 1) { cv::cvtColor(input, localFrame, cv::COLOR_GRAY2BGR); } const cv::Mat& frame = (input.channels() == 1) ? localFrame : input; double msColorConvert = dbg ? elapsed() : 0; const int frameWidth = frame.cols; const int frameHeight = frame.rows; // --- Step 1: LP Detection --- cv::Rect roi(0, 0, 0, 0); std::vector lprOutput = this->_lpDetector->RunStaticInference(frame, roi, cameraId); double msLPDetect = dbg ? elapsed() : 0; int numPlates = (int)lprOutput.size(); double totalOcrMs = 0, totalValidateMs = 0, totalColourMs = 0, totalCropMs = 0; int ocrCount = 0, validCount = 0, colourCount = 0; if (!lprOutput.empty()) { output.reserve(lprOutput.size()); constexpr int padding = 10; // --- Compute display→full-res scale (once per frame, cheap) --- float scaleX = 1.f, scaleY = 1.f; { auto* gpuData = tl_currentGpuFrame(); if (gpuData && gpuData->width > frame.cols && gpuData->height > frame.rows) { scaleX = static_cast(gpuData->width) / frame.cols; scaleY = static_cast(gpuData->height) / frame.rows; } } for (auto& lprObject : lprOutput) { const cv::Rect& box = lprObject.box; // --- Step 2: Crop LP region --- auto tCrop = dbg ? Clock::now() : Clock::time_point{}; cv::Mat lprImage; // Try GPU NV12 crop (NVIDIA decode: NV12 still in GPU VRAM) if (scaleX > 1.f) { auto cropResult = _nv12Helper.tryNV12CropToBGR( frame, 0, box, padding, scaleX, scaleY, this->_logger, "LPR"); if (cropResult.succeeded) lprImage = cropResult.bgrCrop; } // Fallback: crop from display-res frame if (lprImage.empty()) { const int x1 = std::max(0, box.x - padding); const int y1 = std::max(0, box.y - padding); const int x2 = std::min(frameWidth, box.x + box.width + padding); const int y2 = std::min(frameHeight, box.y + box.height + padding); const int width = x2 - x1; const int height = y2 - y1; if (width <= padding || height <= padding) { continue; } lprImage = frame(cv::Rect(x1, y1, width, height)).clone(); } if (dbg) totalCropMs += std::chrono::duration(Clock::now() - tCrop).count(); // --- Step 3: OCR inference --- auto tOcr = dbg ? Clock::now() : Clock::time_point{}; std::string ocrText = DetectLicensePlateString(lprImage, cameraId); if (dbg) { totalOcrMs += std::chrono::duration(Clock::now() - tOcr).count(); ocrCount++; } if (ocrText.empty()) { continue; } // --- Step 4: Plate validation --- auto tValidate = dbg ? Clock::now() : Clock::time_point{}; lprObject.cameraId = cameraId; lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows); if (shouldUseALPRChecker(cv::Size(input.cols, input.rows), cameraId)) { lprObject.className = alprChecker.checkPlateByTrackId(cameraId, ocrText, lprObject.trackId); } else { lprObject.className = ocrText; } if (dbg) { totalValidateMs += std::chrono::duration(Clock::now() - tValidate).count(); } if (lprObject.className.empty()) { continue; } validCount++; // --- Step 5: Colour classification (cached) --- auto tColour = dbg ? Clock::now() : Clock::time_point{}; std::string colour = DetectLPColourCached(lprImage, cameraId, lprObject.className); if (!colour.empty()) { lprObject.extraInfo = "color:" + colour; } if (dbg) { totalColourMs += std::chrono::duration(Clock::now() - tColour).count(); colourCount++; } output.push_back(std::move(lprObject)); } } // --- Step 6: Serialize results --- auto tJson = dbg ? Clock::now() : Clock::time_point{}; lprResult = VectorDetectionToJsonString(output); double msJson = dbg ? std::chrono::duration(Clock::now() - tJson).count() : 0; // --- Log full pipeline breakdown --- if (dbg) { double msTotal = std::chrono::duration(Clock::now() - t0).count(); char buf[1024]; snprintf(buf, sizeof(buf), "[DEBUG] %s | ColorCvt=%.1fms LPDetect=%.1fms (plates=%d) " "Crop=%.1fms OCR=%.1fms (x%d) Validate=%.1fms Colour=%.1fms (x%d) " "JSON=%.1fms | TOTAL=%.1fms Output=%d", cameraId.c_str(), msColorConvert, msLPDetect, numPlates, totalCropMs, totalOcrMs, ocrCount, totalValidateMs, totalColourMs, colourCount, msJson, msTotal, (int)output.size()); _logger.LogInfo("ANSALPR_OD::Inference", buf, __FILE__, __LINE__); } return true; } catch (const std::exception& e) { lprResult = VectorDetectionToJsonString(output); this->_logger.LogFatal("ANSALPR_OD::Inference", e.what(), __FILE__, __LINE__); return false; } } bool ANSALPR_OD::Inference(const cv::Mat& input, const std::vector & Bbox, std::string& lprResult) { // No coarse _mutex — delegates to Inference(input, Bbox, lprResult, cameraId) if (input.empty()) return false; if ((input.cols < 5) || (input.rows < 5)) return false; return Inference(input, Bbox, lprResult, "CustomCam"); } bool ANSALPR_OD::Inference(const cv::Mat& input, const std::vector& Bbox,std::string& lprResult, const std::string& cameraId) { // No coarse _mutex — sub-components have their own fine-grained locks. // Early validation if (!_licenseValid) { this->_logger.LogError("ANSALPR_OD::Inference", "Invalid license", __FILE__, __LINE__); lprResult.clear(); return false; } if (!valid) { this->_logger.LogError("ANSALPR_OD::Inference", "Invalid model", __FILE__, __LINE__); lprResult.clear(); return false; } if (!_isInitialized) { this->_logger.LogError("ANSALPR_OD::Inference", "Model is not initialized", __FILE__, __LINE__); lprResult.clear(); return false; } if (input.empty()) { this->_logger.LogError("ANSALPR_OD::Inference", "Input image is empty", __FILE__, __LINE__); lprResult.clear(); return false; } if (input.cols < 5 || input.rows < 5) { this->_logger.LogError("ANSALPR_OD::Inference", "Input image size is too small", __FILE__, __LINE__); lprResult.clear(); return false; } if (!_lpDetector) { this->_logger.LogFatal("ANSALPR_OD::Inference", "_lpDetector is null", __FILE__, __LINE__); lprResult.clear(); return false; } try { // Convert grayscale to BGR if necessary (use local buffer for thread safety) cv::Mat localFrame; if (input.channels() == 1) { cv::cvtColor(input, localFrame, cv::COLOR_GRAY2BGR); } const cv::Mat& frame = (input.channels() == 1) ? localFrame : input; const int frameWidth = frame.cols; const int frameHeight = frame.rows; constexpr int padding = 10; // --- Compute display→full-res scale (once per frame, cheap) --- float scaleX2 = 1.f, scaleY2 = 1.f; { auto* gpuData = tl_currentGpuFrame(); if (gpuData && gpuData->width > frame.cols && gpuData->height > frame.rows) { scaleX2 = static_cast(gpuData->width) / frame.cols; scaleY2 = static_cast(gpuData->height) / frame.rows; } } std::vector detectedObjects; if (!Bbox.empty()) { // Process each bounding box region detectedObjects.reserve(Bbox.size()); for (const auto& bbox : Bbox) { const int x1c = std::max(0, bbox.x); const int y1c = std::max(0, bbox.y); const int cropWidth = std::min(frameWidth - x1c, bbox.width); const int cropHeight = std::min(frameHeight - y1c, bbox.height); if (cropWidth < 5 || cropHeight < 5) { continue; } cv::Rect objectPos(x1c, y1c, cropWidth, cropHeight); cv::Mat croppedObject = frame(objectPos); std::vector lprOutput = _lpDetector->RunInference(croppedObject, cameraId); for (size_t _di = 0; _di < lprOutput.size(); ++_di) { ANS_DBG("ALPR_Track", "cam=%s bbox det[%zu] tid=%d box=(%d,%d,%d,%d) conf=%.2f", cameraId.c_str(), _di, lprOutput[_di].trackId, lprOutput[_di].box.x, lprOutput[_di].box.y, lprOutput[_di].box.width, lprOutput[_di].box.height, lprOutput[_di].confidence); } for (auto& lprObject : lprOutput) { const cv::Rect& box = lprObject.box; // Calculate padded region within cropped image const int x1 = std::max(0, box.x - padding); const int y1 = std::max(0, box.y - padding); const int x2 = std::min(cropWidth, box.x + box.width + padding); const int y2 = std::min(cropHeight, box.y + box.height + padding); // Adjust to original frame coordinates lprObject.box.x = std::max(0, x1c + x1); lprObject.box.y = std::max(0, y1c + y1); lprObject.box.width = std::min(frameWidth - lprObject.box.x, x2 - x1); lprObject.box.height = std::min(frameHeight - lprObject.box.y, y2 - y1); if (lprObject.box.width <= padding || lprObject.box.height <= padding) { continue; } lprObject.cameraId = cameraId; lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows); // Crop from full-res NV12 on GPU if available, otherwise display-res cv::Mat lprImage; if (scaleX2 > 1.f) { auto cropResult = _nv12Helper.tryNV12CropToBGR( frame, 0, lprObject.box, 0, scaleX2, scaleY2, this->_logger, "LPR"); if (cropResult.succeeded) lprImage = cropResult.bgrCrop; } if (lprImage.empty()) lprImage = frame(lprObject.box); cv::Mat alignedLPR = enhanceForOCR(lprImage); std::string ocrText = DetectLicensePlateString(alignedLPR, cameraId); if (ocrText.empty()) { continue; } if (shouldUseALPRChecker(cv::Size(input.cols, input.rows), cameraId)) { lprObject.className = alprChecker.checkPlateByTrackId(cameraId, ocrText, lprObject.trackId); } else { lprObject.className = ocrText; } if (lprObject.className.empty()) { continue; } std::string colour = DetectLPColourCached(lprImage, cameraId, lprObject.className); if (!colour.empty()) { lprObject.extraInfo = "color:" + colour; } detectedObjects.push_back(std::move(lprObject)); } } } else { // No bounding boxes - run on full frame std::vector lprOutput = _lpDetector->RunInference(frame, cameraId); for (size_t _di = 0; _di < lprOutput.size(); ++_di) { ANS_DBG("ALPR_Track", "cam=%s full det[%zu] tid=%d box=(%d,%d,%d,%d) conf=%.2f", cameraId.c_str(), _di, lprOutput[_di].trackId, lprOutput[_di].box.x, lprOutput[_di].box.y, lprOutput[_di].box.width, lprOutput[_di].box.height, lprOutput[_di].confidence); } detectedObjects.reserve(lprOutput.size()); for (auto& lprObject : lprOutput) { const cv::Rect& box = lprObject.box; // Calculate padded region const int x1 = std::max(0, box.x - padding); const int y1 = std::max(0, box.y - padding); const int width = std::min(frameWidth - x1, box.width + 2 * padding); const int height = std::min(frameHeight - y1, box.height + 2 * padding); if (width <= padding || height <= padding) { continue; } lprObject.cameraId = cameraId; lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows); // Crop from full-res NV12 on GPU if available, otherwise display-res cv::Rect lprPos(x1, y1, width, height); cv::Mat lprImage; if (scaleX2 > 1.f) { auto cropResult = _nv12Helper.tryNV12CropToBGR( frame, 0, lprPos, 0, scaleX2, scaleY2, this->_logger, "LPR"); if (cropResult.succeeded) lprImage = cropResult.bgrCrop; } if (lprImage.empty()) lprImage = frame(lprPos); cv::Mat alignedLPR = enhanceForOCR(lprImage); std::string rawText = DetectLicensePlateString(alignedLPR, cameraId); if (shouldUseALPRChecker(cv::Size(input.cols, input.rows), cameraId)) { lprObject.className = alprChecker.checkPlateByTrackId(cameraId, rawText, lprObject.trackId); } else { lprObject.className = rawText; } if (lprObject.className.empty()) { continue; } std::string colour = DetectLPColourCached(lprImage, cameraId, lprObject.className); if (!colour.empty()) { lprObject.extraInfo = "color:" + colour; } detectedObjects.push_back(std::move(lprObject)); } } // Deduplicate: same plate text should not appear on multiple vehicles // Note: in Bbox mode, internal LP trackIds overlap across crops, so // dedup uses plate bounding box position (via Object::box) to distinguish. // The ensureUniquePlateText method handles this by plate text grouping. if (shouldUseALPRChecker(cv::Size(input.cols, input.rows), cameraId)) { ensureUniquePlateText(detectedObjects, cameraId); } lprResult = VectorDetectionToJsonString(detectedObjects); return true; } catch (const std::exception& e) { lprResult.clear(); this->_logger.LogFatal("ANSALPR_OD::Inference", e.what(), __FILE__, __LINE__); return false; } } int ANSALPR_OD::findSubstringIndex(const std::string& str) { //std::lock_guard lock(_mutex); try { // List of substrings to search for std::string substrings[] = { "NN", "CV", "NG", "QT" }; // Iterate through each substring for (const std::string& sub : substrings) { // Use std::string::find to search for the substring in the given string std::size_t pos = str.find(sub); // If the substring is found, return the index if (pos != std::string::npos) { return static_cast(pos); // Cast to int and return the index } } // If none of the substrings is found, return -1 return -1; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::findSubstringIndex", e.what(), __FILE__, __LINE__); return -1; } } char ANSALPR_OD::fixLPDigit(char c) { //std::lock_guard lock(_mutex); try { switch (c) { case 'b': return '6'; case 'c': return '0'; case 'f': case 't': return '4'; case 'j': case 'i': case 'l': return '1'; case 's': return '5'; case 'g': case 'q': case 'y': return '9'; case 'o': return '0'; default: return c; // If the character is not a letter to convert, return it unchanged } } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::fixLPDigit", e.what(), __FILE__, __LINE__); return c; } } //only accept these letters: A, B, C, D, E, F, G, H, K, L, M, N, P, S, T, U, V, X, Y, Z // I, J, O, Q, R, W char ANSALPR_OD::convertDigitToLetter(char c) { //std::lock_guard lock(_mutex); try { switch (c) { case '0': case 'o': case 'O': case 'Q': return 'C'; // '0' is typically mapped to 'O' or 'C', choosing 'O' to match letter set case '1': case 'I': case 'i': case 'l': case 'J': return 'L'; // '1' is commonly confused with 'I' case '2': case 'z': return 'Z'; // '2' resembles 'Z' in some fonts case '3': return 'E'; // '3' can resemble 'E' in some cases case '4': return 'A'; // '4' can resemble 'A' or 'H', choosing 'A' case '5': case 's': return 'S'; // '5' looks similar to 'S' case '6': case 'g': return 'G'; // '6' resembles 'G' case '7': return 'T'; // '7' is often confused with 'T' case '8': case 'b': return 'B'; // '8' resembles 'B' case '9': case 'R': return 'P'; // '9' is close to 'P' case 'W': case 'w': return 'V'; // 'W' is close to 'V' default: return c; // If the character is not a digit to convert, return it unchanged } } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::convertDigitToLetter", e.what(), __FILE__, __LINE__); return c; } } char ANSALPR_OD::convertLetterToDigit(char c) { // std::lock_guard lock(_mutex); try { switch (c) { // Convert common letter confusions with digits case 'B': case 'b': // Adding lowercase 'b' to match common mistypes return '8'; case 'I': case 'i': case 'J': // Capital 'J' can also resemble '1' case 'j': case 'L': case 'l': return '1'; case 'S': case 's': return '5'; case 'G': case 'g': // Adding lowercase 'g' for better matching return '6'; case 'O': case 'o': case 'Q': // 'Q' can also be misread as '0' case 'U': case 'u': // Adding lowercase 'u' as it resembles '0' return '0'; case 'T': // Capital 'T' sometimes looks like '7' return '7'; case 'F': case 'f': case 't': return '4'; case 'Y': // Capital 'Y' may resemble '9' case 'y': case 'q': return '9'; default: return '0'; // If no conversion, return the character unchanged } } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::convertLetterToDigit", e.what(), __FILE__, __LINE__); return c; } } // Function to convert string to digits, skipping conversion if the character is already a digit std::string ANSALPR_OD::convertStringToDigits(const std::string& input) { // std::lock_guard lock(_mutex); try { std::string result; for (char c : input) { if (std::isdigit(c)) { result += c; // Skip conversion if the character is a digit } else { result += convertLetterToDigit(c); // Convert if it's a letter } } return result; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::convertStringToDigits", e.what(), __FILE__, __LINE__); return input; } } // Function to convert string to letters, skipping conversion if the character is already a letter std::string ANSALPR_OD::convertStringToLetters(const std::string& input) { //std::lock_guard lock(_mutex); try { std::string result; for (char c : input) { if (std::isalpha(c)) { result += c; // Skip conversion if the character is already a letter } else { result += convertDigitToLetter(c); // Convert if it's a digit } } return result; } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::convertStringToLetters", e.what(), __FILE__, __LINE__); return input; } } int ANSALPR_OD::searchDiplomacyLP(const std::string& input) { //std::lock_guard lock(_mutex); // List of substrings to search for try { std::string substrings[] = { "NN", "NG", "CV", "QT" }; // Initialize index to -1 (not found) int foundIndex = -1; // Loop through the substrings for (const auto& sub : substrings) { // Find the index of the current substring size_t index = input.find(sub); // If the substring is found and either no other substrings have been found, // or this substring occurs at an earlier position, update foundIndex. if (index != std::string::npos && (foundIndex == -1 || index < foundIndex)) { foundIndex = index; } } return foundIndex; // If none are found, returns -1 } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::searchDiplomacyLP", e.what(), __FILE__, __LINE__); return -1; } } bool ANSALPR_OD::ValidateVNMotobikeLP(const std::string& input) { // std::lock_guard lock(_mutex); // Search for the string in the list auto it = std::find(ValidVNMotobikeList.begin(), ValidVNMotobikeList.end(), input); // Check if found if (it != ValidVNMotobikeList.end()) { return true; } else { return false; } } bool ANSALPR_OD::ValidateVNCarLP(const std::string& input) { // std::lock_guard lock(_mutex); try { // Search for the string in the list auto it = std::find(ValidVNCarList.begin(), ValidVNCarList.end(), input); // Check if found if (it != ValidVNCarList.end()) { return true; } else { return false; } } catch (std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::ValidateVNCarLP", e.what(), __FILE__, __LINE__); return false; } } cv::Mat ANSALPR_OD::alignPlateForOCR(const cv::Mat& fullImage, const cv::Rect& bbox) { try { const cv::Rect safeBox = bbox & cv::Rect(0, 0, fullImage.cols, fullImage.rows); if (safeBox.width < 10 || safeBox.height < 10) { return fullImage(safeBox).clone(); } cv::Mat roi = fullImage(safeBox); // Convert to grayscale and create binary image cv::Mat gray; cv::cvtColor(roi, gray, cv::COLOR_BGR2GRAY); cv::Mat binary; cv::adaptiveThreshold(gray, binary, 255, cv::ADAPTIVE_THRESH_MEAN_C, cv::THRESH_BINARY_INV, 15, 10); std::vector> contours; cv::findContours(binary, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE); if (contours.empty()) { return enhanceAndDebug(roi); } // Find best contour closest to center const cv::Point2f roiCenter(static_cast(roi.cols) / 2.0f, static_cast(roi.rows) / 2.0f); const float minWidth = roi.cols * 0.5f; const float minHeight = roi.rows * 0.5f; constexpr float minAreaRatio = 0.3f; float minDist = std::numeric_limits::max(); int bestIdx = -1; for (size_t i = 0; i < contours.size(); ++i) { cv::RotatedRect rect = cv::minAreaRect(contours[i]); const float width = rect.size.width; const float height = rect.size.height; if (width < minWidth || height < minHeight) { continue; } const float areaRect = width * height; const float areaContour = static_cast(cv::contourArea(contours[i])); if (areaContour / areaRect < minAreaRatio) { continue; } const float dist = cv::norm(rect.center - roiCenter); if (dist < minDist) { minDist = dist; bestIdx = static_cast(i); } } if (bestIdx == -1) { return enhanceAndDebug(roi); } // Align using best rotated rect cv::RotatedRect bestRect = cv::minAreaRect(contours[bestIdx]); float angle = bestRect.angle; if (bestRect.size.width < bestRect.size.height) { angle += 90.0f; std::swap(bestRect.size.width, bestRect.size.height); } angle = std::clamp(angle, -45.0f, 45.0f); // Rotate the image const cv::Point2f rotationCenter(roi.cols / 2.0f, roi.rows / 2.0f); cv::Mat rotationMatrix = cv::getRotationMatrix2D(rotationCenter, angle, 1.0); cv::Mat rotated; cv::warpAffine(roi, rotated, rotationMatrix, roi.size(), cv::INTER_LINEAR, cv::BORDER_REPLICATE); // Transform rect center after rotation const double* rotData = rotationMatrix.ptr(0); const cv::Point2f newCenter( static_cast(rotData[0] * bestRect.center.x + rotData[1] * bestRect.center.y + rotData[2]), static_cast(rotData[3] * bestRect.center.x + rotData[4] * bestRect.center.y + rotData[5]) ); // Apply small padding and crop constexpr int padding = 2; const cv::Size paddedSize( std::min(rotated.cols, static_cast(bestRect.size.width) + 2 * padding), std::min(rotated.rows, static_cast(bestRect.size.height) + 2 * padding) ); cv::Mat rawCropped; cv::getRectSubPix(rotated, paddedSize, newCenter, rawCropped); cv::Mat cropped = enhanceForOCR(rawCropped); #ifdef FNS_DEBUG showDebugComparison(roi, cropped, contours, bestIdx, bestRect); #endif return cropped; } catch (const std::exception& e) { this->_logger.LogError("ANSALPR_OD::alignPlateForOCR", std::string("Exception: ") + e.what(), __FILE__, __LINE__); return fullImage(bbox & cv::Rect(0, 0, fullImage.cols, fullImage.rows)).clone(); } } #ifdef FNS_DEBUG void ANSALPR_OD::showDebugComparison(const cv::Mat& roi, const cv::Mat& processed, const std::vector>& contours, int bestIdx, const cv::RotatedRect& bestRect) { try { cv::Mat debugRoi = roi.clone(); if (bestIdx >= 0) { cv::drawContours(debugRoi, contours, bestIdx, cv::Scalar(0, 255, 0), 1); cv::Point2f points[4]; bestRect.points(points); for (int j = 0; j < 4; ++j) { cv::line(debugRoi, points[j], points[(j + 1) % 4], cv::Scalar(255, 0, 0), 1); } } cv::Mat debugLeft, debugRight; cv::resize(debugRoi, debugLeft, cv::Size(240, 80)); cv::resize(processed, debugRight, cv::Size(240, 80)); cv::Mat combined; cv::hconcat(debugLeft, debugRight, combined); cv::putText(combined, "Raw", cv::Point(10, 15), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1); cv::putText(combined, "Aligned", cv::Point(250, 15), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 0, 0), 1); cv::imshow("LPR Cropped + Rotated", combined); cv::waitKey(1); } catch (const std::exception& e) { std::cerr << "LPR Debug Error: " << e.what() << std::endl; } } #endif cv::Mat ANSALPR_OD::enhanceAndDebug(const cv::Mat& roi) { cv::Mat enhanced = enhanceForOCR(roi); #ifdef FNS_DEBUG showDebugComparison(roi, enhanced, {}, -1, cv::RotatedRect()); #endif return enhanced; } cv::Mat ANSALPR_OD::enhanceForOCR(const cv::Mat& plateROIOriginal) { if (plateROIOriginal.empty()) { this->_logger.LogError("ANSALPR_OD::enhanceForOCR", "plateROI is empty", __FILE__, __LINE__); return cv::Mat(); } // Step 1: Upscale for OCR clarity cv::Mat plateROI; cv::resize(plateROIOriginal, plateROI, cv::Size(), 2.0, 2.0, cv::INTER_LANCZOS4); // Step 2: Grayscale cv::Mat gray; if (plateROI.channels() == 3) { cv::cvtColor(plateROI, gray, cv::COLOR_BGR2GRAY); } else { gray = plateROI; } // Step 3: Gentle denoise to preserve edges cv::Mat denoised; cv::bilateralFilter(gray, denoised, 7, 50, 50); // Step 4: Unsharp masking cv::Mat blurred; cv::GaussianBlur(denoised, blurred, cv::Size(0, 0), 1.5); cv::Mat unsharp; cv::addWeighted(denoised, 1.8, blurred, -0.8, 0, unsharp); // Step 5: CLAHE contrast enhancement (thread-local for thread safety) thread_local cv::Ptr tl_clahe = cv::createCLAHE(4.0, cv::Size(8, 8)); cv::Mat contrastEnhanced; tl_clahe->apply(unsharp, contrastEnhanced); // Step 6: Laplacian edge sharpening cv::Mat lap; cv::Laplacian(contrastEnhanced, lap, CV_16S, 3); cv::Mat lapAbs; cv::convertScaleAbs(lap, lapAbs); cv::Mat sharpened; cv::addWeighted(contrastEnhanced, 1.2, lapAbs, -0.3, 0, sharpened); // Step 7: Convert back to BGR for OCR cv::Mat ocrInput; cv::cvtColor(sharpened, ocrInput, cv::COLOR_GRAY2BGR); return ocrInput; } // Batch Inference std::vector ANSALPR_OD::RunInference(const cv::Mat& input, const std::string& cameraId) { // Read-only validation without lock (immutable after initialization) if (!_licenseValid || !valid || !_isInitialized) { this->_logger.LogWarn("ANSALPR_OD::RunInference", "Invalid state: license=" + std::to_string(_licenseValid) + " valid=" + std::to_string(valid) + " init=" + std::to_string(_isInitialized), __FILE__, __LINE__); return {}; } if (input.empty() || input.cols < 5 || input.rows < 5) { this->_logger.LogWarn("ANSALPR_OD::RunInference", "Skipped: input too small (" + std::to_string(input.cols) + "x" + std::to_string(input.rows) + ")", __FILE__, __LINE__); return {}; } // Pointer checks (these should be immutable after initialization) if (!this->_lpDetector || !this->_ocrDetector) { this->_logger.LogFatal("ANSALPR_OD::RunInference", "Detector instances are null", __FILE__, __LINE__); return {}; } try { // Use local buffer instead of shared _frameBuffer cv::Mat frame; if (input.channels() == 1) { cv::cvtColor(input, frame, cv::COLOR_GRAY2BGR); } else { frame = input; // No copy, just reference } const int frameWidth = frame.cols; const int frameHeight = frame.rows; // Use local variable instead of shared _detectedArea cv::Rect detectedArea(0, 0, frameWidth, frameHeight); if (detectedArea.width <= 50 || detectedArea.height <= 50) { return {}; } // Run license plate detection (should be thread-safe internally) cv::Mat activeFrame = frame(detectedArea); std::vector lprOutput = _lpDetector->RunInference(activeFrame, cameraId); for (size_t _di = 0; _di < lprOutput.size(); ++_di) { ANS_DBG("ALPR_Track", "cam=%s batch det[%zu] tid=%d box=(%d,%d,%d,%d) conf=%.2f", cameraId.c_str(), _di, lprOutput[_di].trackId, lprOutput[_di].box.x, lprOutput[_di].box.y, lprOutput[_di].box.width, lprOutput[_di].box.height, lprOutput[_di].confidence); } if (lprOutput.empty()) { return {}; } // Prepare batch - pre-allocate and use move semantics std::vector alignedLPRBatch; std::vector validIndices; alignedLPRBatch.reserve(lprOutput.size()); validIndices.reserve(lprOutput.size()); for (size_t i = 0; i < lprOutput.size(); ++i) { const cv::Rect& box = lprOutput[i].box; // Calculate cropped region with bounds checking const int x1 = std::max(0, box.x); const int y1 = std::max(0, box.y); const int x2 = std::min(frameWidth, box.x + box.width); const int y2 = std::min(frameHeight, box.y + box.height); const int width = x2 - x1; const int height = y2 - y1; if (width <= 0 || height <= 0) { continue; } cv::Rect lprPos(x1, y1, width, height); alignedLPRBatch.emplace_back(frame(lprPos)); // Use emplace_back validIndices.push_back(i); } if (alignedLPRBatch.empty()) { return {}; } // Run OCR first, then use cached colour detection. // Colour caching by plate text eliminates ~95% of LPC inferences // (plate colour doesn't change frame-to-frame). std::vector ocrTextBatch = DetectLicensePlateStringBatch(alignedLPRBatch, cameraId); // Build output — colour detection uses cache keyed by stabilized plate text std::vector output; output.reserve(validIndices.size()); for (size_t i = 0; i < validIndices.size(); ++i) { const size_t origIdx = validIndices[i]; const std::string& ocrText = ocrTextBatch[i]; if (ocrText.empty()) { continue; } Object lprObject = lprOutput[origIdx]; lprObject.cameraId = cameraId; // Stabilize OCR text through ALPRChecker (hybrid trackId + Levenshtein fallback) if (shouldUseALPRChecker(cv::Size(input.cols, input.rows), cameraId)) { lprObject.className = alprChecker.checkPlateByTrackId(cameraId, ocrText, lprObject.trackId); } else { lprObject.className = ocrText; } if (lprObject.className.empty()) { continue; } // Colour detection with cache — only runs inference on first // occurrence of each plate text, subsequent frames get 0ms cache hit std::string colour = DetectLPColourCached(alignedLPRBatch[i], cameraId, lprObject.className); if (!colour.empty()) { lprObject.extraInfo = "color:" + colour; } output.push_back(std::move(lprObject)); } // Deduplicate: if two trackIds claim the same plate text, keep the one // with the higher accumulated score to prevent plate flickering if (shouldUseALPRChecker(cv::Size(input.cols, input.rows), cameraId)) { ensureUniquePlateText(output, cameraId); } return output; } catch (const cv::Exception& e) { this->_logger.LogFatal("ANSALPR_OD::RunInference", std::string("OpenCV Exception: ") + e.what(), __FILE__, __LINE__); } catch (const std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::RunInference", e.what(), __FILE__, __LINE__); } catch (...) { this->_logger.LogFatal("ANSALPR_OD::RunInference", "Unknown exception occurred", __FILE__, __LINE__); } return {}; } std::vector ANSALPR_OD::DetectLPColourDetectorBatch(const std::vector& lprROIs, const std::string& cameraId) { // Early validation - no lock needed for immutable config if (_lpColourModelConfig.detectionScoreThreshold <= 0.0f || !_lpColourDetector) { return std::vector(lprROIs.size(), ""); } try { if (lprROIs.empty()) { return {}; } const size_t batchSize = lprROIs.size(); // Filter out empty ROIs — just a cheap .empty() check, no need for threads std::vector validROIs; std::vector validIndices; validROIs.reserve(batchSize); validIndices.reserve(batchSize); for (size_t i = 0; i < batchSize; ++i) { if (!lprROIs[i].empty()) { validROIs.push_back(lprROIs[i]); validIndices.push_back(i); } } if (validROIs.empty()) { return std::vector(batchSize, ""); } // Run batch colour detection (GPU-accelerated, already optimized) std::vector> colourBatchOutputs = _lpColourDetector->RunInferencesBatch(validROIs, cameraId); if (colourBatchOutputs.size() != validROIs.size()) { this->_logger.LogError("ANSALPR_OD::DetectLPColourDetectorBatch", "Colour detector batch size mismatch", __FILE__, __LINE__); return std::vector(batchSize, ""); } // Prepare results vector (initialize all to empty) std::vector results(batchSize); // Process results in parallel for large batches const size_t validSize = colourBatchOutputs.size(); if (validSize > 10) { const unsigned int hwThreads = std::thread::hardware_concurrency(); const unsigned int numThreads = std::min(hwThreads > 0 ? hwThreads : 4, static_cast(validSize)); const size_t chunkSize = (validSize + numThreads - 1) / numThreads; std::vector> futures; futures.reserve(numThreads); for (unsigned int t = 0; t < numThreads; ++t) { const size_t startIdx = t * chunkSize; const size_t endIdx = std::min(startIdx + chunkSize, validSize); if (startIdx >= validSize) break; futures.push_back(std::async(std::launch::async, [&colourBatchOutputs, &validIndices, &results, startIdx, endIdx, this]() { const float threshold = _lpColourModelConfig.detectionScoreThreshold; for (size_t i = startIdx; i < endIdx; ++i) { if (colourBatchOutputs[i].empty()) { continue; } // Find detection with highest confidence above threshold float maxConfidence = threshold; std::string bestClassName; for (const auto& detection : colourBatchOutputs[i]) { if (detection.confidence > maxConfidence) { maxConfidence = detection.confidence; bestClassName = detection.className; } } if (!bestClassName.empty()) { results[validIndices[i]] = bestClassName; } } } )); } // Wait for all processing to complete for (auto& future : futures) { try { if (future.wait_for(std::chrono::seconds(ASYNC_TIMEOUT_SECONDS)) == std::future_status::timeout) { this->_logger.LogError("ANSALPR_OD::DetectLPColourDetectorBatch", "Async colour processing timed out (" + std::to_string(ASYNC_TIMEOUT_SECONDS) + "s)", __FILE__, __LINE__); continue; } future.get(); } catch (const std::exception& e) { this->_logger.LogError("ANSALPR_OD::DetectLPColourDetectorBatch", std::string("Async colour processing failed: ") + e.what(), __FILE__, __LINE__); } } } else { // Sequential for small batches const float threshold = _lpColourModelConfig.detectionScoreThreshold; for (size_t i = 0; i < validSize; ++i) { if (colourBatchOutputs[i].empty()) { continue; } // Find detection with highest confidence above threshold float maxConfidence = threshold; std::string bestClassName; for (const auto& detection : colourBatchOutputs[i]) { if (detection.confidence > maxConfidence) { maxConfidence = detection.confidence; bestClassName = detection.className; } } if (!bestClassName.empty()) { results[validIndices[i]] = bestClassName; } } } return results; } catch (const std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::DetectLPColourDetectorBatch", e.what(), __FILE__, __LINE__); return std::vector(lprROIs.size(), ""); } } std::vector ANSALPR_OD::DetectLicensePlateStringBatch(const std::vector& lprROIs, const std::string& cameraId) { if (lprROIs.empty()) { return {}; } const size_t batchSize = lprROIs.size(); std::vector results(batchSize); try { // Prepare grayscale batch — cvtColor on small LP crops is microseconds, // thread spawn overhead dominates, so always run sequentially. std::vector grayBatch(batchSize); for (size_t i = 0; i < batchSize; ++i) { const cv::Mat& lprROI = lprROIs[i]; if (lprROI.empty()) continue; if (lprROI.channels() == 3) { cv::cvtColor(lprROI, grayBatch[i], cv::COLOR_BGR2GRAY); } else if (lprROI.channels() == 1) { grayBatch[i] = lprROI; } } // Run batch OCR inference (GPU-accelerated, already optimized) std::vector> ocrBatchOutputs = _ocrDetector->RunInferencesBatch(grayBatch, cameraId); if (ocrBatchOutputs.size() != batchSize) { this->_logger.LogWarn("ANSALPR_OD::DetectLicensePlateStringBatch", "Skipped: OCR batch size mismatch", __FILE__, __LINE__); return std::vector(batchSize, ""); } // Process OCR results — ProcessSingleOCRResult is pure CPU math // on ~5-20 characters per plate, so thread overhead dominates. for (size_t i = 0; i < batchSize; ++i) { results[i] = ProcessSingleOCRResult(ocrBatchOutputs[i]); } return results; } catch (const std::exception& e) { this->_logger.LogFatal("ANSALPR_OD::DetectLicensePlateStringBatch", e.what(), __FILE__, __LINE__); return std::vector(batchSize, ""); } } // New helper function - thread-safe, no shared state std::string ANSALPR_OD::ProcessSingleOCRResult(const std::vector& ocrOutput) { if (ocrOutput.empty()) { return ""; } // Remove duplicates using spatial hashing for O(n) instead of O(n^2) std::unordered_map> spatialHash; spatialHash.reserve(ocrOutput.size()); std::vector uniqueOutput; uniqueOutput.reserve(ocrOutput.size()); for (size_t i = 0; i < ocrOutput.size(); ++i) { const auto& obj = ocrOutput[i]; const int gridX = static_cast(obj.box.x / DUPLICATE_GRID_SIZE); const int gridY = static_cast(obj.box.y / DUPLICATE_GRID_SIZE); const int64_t hashKey = static_cast(gridY) * 100000LL + gridX; bool isDuplicate = false; // Check only nearby cells (current + 8 neighbors) for (int dy = -1; dy <= 1 && !isDuplicate; ++dy) { for (int dx = -1; dx <= 1; ++dx) { const int64_t neighborKey = static_cast(gridY + dy) * 100000LL + (gridX + dx); auto it = spatialHash.find(neighborKey); if (it != spatialHash.end()) { for (size_t idx : it->second) { const auto& unique = uniqueOutput[idx]; if (std::abs(obj.box.x - unique.box.x) < DUPLICATE_DIST_THRESHOLD && std::abs(obj.box.y - unique.box.y) < DUPLICATE_DIST_THRESHOLD) { isDuplicate = true; break; } } if (isDuplicate) break; } } } if (!isDuplicate) { spatialHash[hashKey].push_back(uniqueOutput.size()); uniqueOutput.push_back(obj); } } if (uniqueOutput.empty()) { return ""; } if (uniqueOutput.size() == 1) { return AnalyseLicensePlateText(uniqueOutput[0].className); } // ---- Character centers and average height ---- float avgHeight = 0.0f; int n = static_cast(uniqueOutput.size()); std::vector cxs(n), cys(n); for (int i = 0; i < n; ++i) { cxs[i] = uniqueOutput[i].box.x + uniqueOutput[i].box.width * 0.5f; cys[i] = uniqueOutput[i].box.y + uniqueOutput[i].box.height * 0.5f; avgHeight += uniqueOutput[i].box.height; } avgHeight /= static_cast(n); // ---- Mean center ---- float meanX = 0.0f, meanY = 0.0f; for (int i = 0; i < n; ++i) { meanX += cxs[i]; meanY += cys[i]; } meanX /= n; meanY /= n; // PCA to find the plate's reading direction. // Robust to large tilt angles unlike OLS regression. float cov_xx = 0.0f, cov_xy = 0.0f, cov_yy = 0.0f; for (int i = 0; i < n; ++i) { float dx = cxs[i] - meanX; float dy = cys[i] - meanY; cov_xx += dx * dx; cov_xy += dx * dy; cov_yy += dy * dy; } float theta = 0.5f * std::atan2(2.0f * cov_xy, cov_xx - cov_yy); float dirX = std::cos(theta); float dirY = std::sin(theta); if (dirX < 0) { dirX = -dirX; dirY = -dirY; } float perpX = -dirY; float perpY = dirX; // Project centers onto both axes std::vector projAlong(n), projPerp(n); for (int i = 0; i < n; ++i) { float dx = cxs[i] - meanX; float dy = cys[i] - meanY; projAlong[i] = dx * dirX + dy * dirY; projPerp[i] = dx * perpX + dy * perpY; } // Row splitting using perpendicular projections std::vector> perpSorted; perpSorted.reserve(n); for (int i = 0; i < n; ++i) perpSorted.push_back({ projPerp[i], static_cast(i) }); std::sort(perpSorted.begin(), perpSorted.end()); // Find largest gap with validation: // 1. Both groups must have >= 2 chars // 2. Groups must be vertically separated (avgY check) float maxGap = 0.0f; size_t splitIdx = perpSorted.size() / 2; const size_t minGroupSize = ROW_SPLIT_MIN_GROUP_SIZE; for (size_t i = 1; i < perpSorted.size(); ++i) { float gap = perpSorted[i].first - perpSorted[i - 1].first; if (i < minGroupSize || (perpSorted.size() - i) < minGroupSize) continue; if (gap > maxGap) { float avgY_g1 = 0.0f, avgY_g2 = 0.0f; for (size_t j = 0; j < i; ++j) avgY_g1 += cys[perpSorted[j].second]; for (size_t j = i; j < perpSorted.size(); ++j) avgY_g2 += cys[perpSorted[j].second]; avgY_g1 /= static_cast(i); avgY_g2 /= static_cast(perpSorted.size() - i); if (std::abs(avgY_g2 - avgY_g1) > avgHeight * ROW_SPLIT_AVGY_FACTOR) { maxGap = gap; splitIdx = i; } } } if (maxGap < avgHeight * ROW_SPLIT_MIN_GAP_FACTOR) { // Single row - sort by projection along reading direction std::vector> allProj; for (int i = 0; i < n; ++i) allProj.push_back({ projAlong[i], static_cast(i) }); std::sort(allProj.begin(), allProj.end()); std::string ocrText; ocrText.reserve(n); for (const auto& p : allProj) ocrText += uniqueOutput[p.second].className; return AnalyseLicensePlateText(ocrText); } else { // Two rows std::vector> rowIndices(2); for (size_t i = 0; i < perpSorted.size(); ++i) { size_t objIdx = perpSorted[i].second; if (i < splitIdx) rowIndices[0].push_back(objIdx); else rowIndices[1].push_back(objIdx); } // Ensure row 0 is top row (lower average center-Y) float avgY0 = 0.0f, avgY1 = 0.0f; for (auto idx : rowIndices[0]) avgY0 += cys[idx]; for (auto idx : rowIndices[1]) avgY1 += cys[idx]; avgY0 /= static_cast(rowIndices[0].size()); avgY1 /= static_cast(rowIndices[1].size()); if (avgY0 > avgY1) std::swap(rowIndices[0], rowIndices[1]); // Sort each row by projection along reading direction std::string ocrText; ocrText.reserve(n); for (const auto& row : rowIndices) { std::vector> rowProj; for (auto idx : row) rowProj.push_back({ projAlong[idx], idx }); std::sort(rowProj.begin(), rowProj.end()); for (const auto& p : rowProj) ocrText += uniqueOutput[p.second].className; } return AnalyseLicensePlateText(ocrText); } } void ANSALPR_OD::ensureUniquePlateText(std::vector& results, const std::string& cameraId) { std::lock_guard plateLock(_plateIdentitiesMutex); auto& identities = _plateIdentities[cameraId]; // Option B: Auto-detect mode by counting detections. // 1 detection → crop/pipeline mode → return instant result, no accumulated scoring // 2+ detections → full-frame mode → use accumulated scoring for dedup if (results.size() <= 1) { // Still prune stale spatial identities from previous full-frame calls if (!identities.empty()) { constexpr int MAX_UNSEEN_FRAMES = 30; for (auto& id : identities) { id.framesSinceLastSeen++; } for (auto it = identities.begin(); it != identities.end(); ) { if (it->framesSinceLastSeen > MAX_UNSEEN_FRAMES) { it = identities.erase(it); } else { ++it; } } } return; } // --- Full-frame mode: 2+ detections, apply accumulated-score dedup --- // Helper: compute IoU between two rects auto computeIoU = [](const cv::Rect& a, const cv::Rect& b) -> float { int x1 = std::max(a.x, b.x); int y1 = std::max(a.y, b.y); int x2 = std::min(a.x + a.width, b.x + b.width); int y2 = std::min(a.y + a.height, b.y + b.height); if (x2 <= x1 || y2 <= y1) return 0.0f; float intersection = static_cast((x2 - x1) * (y2 - y1)); float unionArea = static_cast(a.area() + b.area()) - intersection; return (unionArea > 0.0f) ? intersection / unionArea : 0.0f; }; // Helper: find matching spatial identity by bounding box overlap auto findSpatialMatch = [&](const cv::Rect& box, const std::string& plateText) -> SpatialPlateIdentity* { for (auto& id : identities) { if (id.plateText == plateText) { // Reconstruct approximate rect from stored center cv::Rect storedRect( static_cast(id.center.x - box.width * 0.5f), static_cast(id.center.y - box.height * 0.5f), box.width, box.height); if (computeIoU(box, storedRect) > PLATE_SPATIAL_MATCH_THRESHOLD) { return &id; } } } return nullptr; }; // Step 1: Build map of plateText → candidate indices std::unordered_map> plateCandidates; for (size_t i = 0; i < results.size(); ++i) { if (results[i].className.empty()) continue; plateCandidates[results[i].className].push_back(i); } // Step 2: Resolve duplicates using spatial accumulated scores for (auto& [plateText, indices] : plateCandidates) { if (indices.size() <= 1) continue; // Find which candidate has the best accumulated score at its location size_t winner = indices[0]; float bestScore = 0.0f; for (size_t idx : indices) { float score = results[idx].confidence; auto* match = findSpatialMatch(results[idx].box, plateText); if (match) { score = match->accumulatedScore + results[idx].confidence; } if (score > bestScore) { bestScore = score; winner = idx; } } // Clear plate text from non-winners for (size_t idx : indices) { if (idx != winner) { results[idx].className.clear(); } } } // Step 3: Update spatial identities — winners accumulate, losers decay constexpr float DECAY_FACTOR = 0.8f; constexpr float MIN_SCORE = 0.1f; constexpr int MAX_UNSEEN_FRAMES = 30; // Age all existing identities for (auto& id : identities) { id.framesSinceLastSeen++; } for (auto& r : results) { if (r.className.empty()) continue; cv::Point2f center( r.box.x + r.box.width * 0.5f, r.box.y + r.box.height * 0.5f); auto* match = findSpatialMatch(r.box, r.className); if (match) { // Same plate at same location — accumulate match->accumulatedScore += r.confidence; match->center = center; // update position match->framesSinceLastSeen = 0; } else { // New plate location — add entry identities.push_back({ center, r.className, r.confidence, 0 }); } } // Decay unseen identities and remove stale ones for (auto it = identities.begin(); it != identities.end(); ) { if (it->framesSinceLastSeen > 0) { it->accumulatedScore *= DECAY_FACTOR; } if (it->accumulatedScore < MIN_SCORE || it->framesSinceLastSeen > MAX_UNSEEN_FRAMES) { it = identities.erase(it); } else { ++it; } } // Step 4: Remove entries with cleared plate text results.erase( std::remove_if(results.begin(), results.end(), [](const Object& o) { return o.className.empty(); }), results.end()); } };