From 0a8aaed215d234bdb09a92c5be4d3f5e7bef6497 Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Sun, 12 Apr 2026 17:16:16 +1000 Subject: [PATCH] Add CPU/GPU gate and support new ANSALPR using OCR --- .claude/settings.local.json | 11 +- CMakeLists.txt | 24 + core/ANSLicensingSystem/ANSLicense.h | 2 +- modules/ANSCV/ANSCVVendorGate.h | 59 ++ modules/ANSCV/ANSFLV.cpp | 7 + modules/ANSCV/ANSMJPEG.cpp | 7 + modules/ANSCV/ANSRTMP.cpp | 7 + modules/ANSCV/ANSRTSP.cpp | 79 +- modules/ANSCV/ANSSRT.cpp | 8 + modules/ANSCV/ANSWebcam.cpp | 17 + modules/ANSCV/dllmain.cpp | 11 + modules/ANSFR/ANSFaceRecognizer.cpp | 37 + modules/ANSFR/dllmain.cpp | 45 +- modules/ANSLPR/ANSALPR_OV.cpp | 843 ---------------------- modules/ANSLPR/ANSLPR.h | 6 + modules/ANSLPR/ANSLPR_OCR.cpp | 665 +++++++++++++++++ modules/ANSLPR/ANSLPR_OCR.h | 91 +++ modules/ANSLPR/ANSLPR_OD.cpp | 267 +++++-- modules/ANSLPR/ANSLPR_OD.h | 24 + modules/ANSLPR/ANSLPR_OV.h | 328 --------- modules/ANSLPR/ANSLPR_RT.cpp | 776 -------------------- modules/ANSLPR/ANSLPR_RT.h | 97 --- modules/ANSLPR/CMakeLists.txt | 2 + modules/ANSLPR/dllmain.cpp | 15 + modules/ANSOCR/ANSOCRBase.h | 2 +- modules/ANSOCR/dllmain.cpp | 34 +- modules/ANSODEngine/ANSODVendorGate.h | 57 ++ modules/ANSODEngine/ANSONNXYOLO.cpp | 152 +++- modules/ANSODEngine/dllmain.cpp | 82 ++- tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp | 281 +++++++- 30 files changed, 1870 insertions(+), 2166 deletions(-) create mode 100644 modules/ANSCV/ANSCVVendorGate.h delete mode 100644 modules/ANSLPR/ANSALPR_OV.cpp create mode 100644 modules/ANSLPR/ANSLPR_OCR.cpp create mode 100644 modules/ANSLPR/ANSLPR_OCR.h delete mode 100644 modules/ANSLPR/ANSLPR_OV.h delete mode 100644 modules/ANSLPR/ANSLPR_RT.cpp delete mode 100644 modules/ANSLPR/ANSLPR_RT.h create mode 100644 modules/ANSODEngine/ANSODVendorGate.h diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 2c4d998..8b8f2d5 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -100,7 +100,16 @@ "Bash(grep -oE \"Faulting[^<]{1,600}|ExceptionCode[^<]{1,100}|FaultingOffset[^<]{1,60}|FaultingModule[^<]{1,200}\" \"C:\\\\Users\\\\nghia\\\\Downloads\\\\Evenlog2.xml\")", "Bash(grep -aoE \"Faulting[^<]{1,600}\" \"C:\\\\Users\\\\nghia\\\\Downloads\\\\Evenlog2.xml\")", "Bash(where python:*)", - "Bash(grep -aoE \"Faulting[^<]{1,700}|ExceptionCode'[^<]{1,60}|FaultingOffset'[^<]{1,60}\" 'C:/Users/nghia/Downloads/Evenlog3.xml')" + "Bash(grep -aoE \"Faulting[^<]{1,700}|ExceptionCode'[^<]{1,60}|FaultingOffset'[^<]{1,60}\" 'C:/Users/nghia/Downloads/Evenlog3.xml')", + "Bash(ls modules/ANSLPR/*.cpp modules/ANSLPR/*.h)", + "Bash(grep -l \"ANSALPR_RT\\\\|ANSALPR_OV\" modules/ANSLPR/*.vcxproj)", + "Bash(grep -l \"anscv_vendor_gate::IsNvidiaGpuAvailable\\\\|ANSCVVendorGate.h\" modules/ANSCV/*.cpp modules/ANSCV/*.h)", + "Bash(grep -l \"CUDA_V2\\\\|CUDAExecutionProvider\" modules/ANSODEngine/*.cpp)", + "Bash(python3 -c \"x=3522082959360; print\\('bytes:', x\\); print\\('/3:', x//3\\); print\\('sqrt\\(x/3\\):', int\\(\\(x//3\\)**0.5\\)\\); print\\('/\\(640*3\\):', x//\\(640*3\\)\\); print\\('/\\(640*640*3\\):', x//\\(640*640*3\\)\\); print\\('hex:', hex\\(x\\)\\)\")", + "Bash(grep -rn \"catch.*cv::Exception\" modules/ANSODEngine/*.cpp)", + "Bash(grep -l \"ANS_DBG\" modules/ANSLPR/*.cpp modules/ANSCV/*.cpp modules/ANSODEngine/*.cpp)", + "Bash(grep -h \"ANS_DBG\\(\\\\\"\" modules/ANSLPR/*.cpp modules/ANSCV/ANSRTSP.cpp modules/ANSODEngine/ANSONNXYOLO.cpp modules/ANSODEngine/ANSRTYOLO.cpp modules/ANSODEngine/NV12PreprocessHelper.cpp)", + "Bash(grep -v \"DNError\\\\|ViewerConfigPath\\\\|Failed to get\\\\|RecursiveDirectory\\\\|qt.qpa\\\\|DispBroker\\\\|SyncInvokeTable\\\\|Created new AppDomain\\\\|Destroying AppDomain\\\\|Trace Start\\\\|ExpandNode\\\\|PublisherMetadata\\\\|at System\\\\.\\\\|at NationalInstruments\\\\|at Mscorlib\\\\|Parameter name\\\\|^[[:space:]]*$\\\\|ArgumentException\\\\|Wrong type\\\\|Concerning target\\\\|Unable to get\\\\|RenderEventToBuffer\\\\|Getting next\\\\|Fetching Next\\\\|Image.Dispose\\\\|Graphics.Dispose\\\\|Image.FromStream\\\\|Inner Exception\\\\|FontFamily\\\\|InitHash\\\\|get_Item\\\\|MethodHandle.InvokeMethod\\\\|RuntimeMethodInfo\\\\|TargetInvocationException\\\\|Hashtable\\\\|LookupControl\\\\|RemoveControl\\\\|CloseInstance\\\\|FreeInstance\\\\|CrossDomainServer\" \"C:/Users/nghia/Downloads/AVNET-8845HS1.log\")" ] } } diff --git a/CMakeLists.txt b/CMakeLists.txt index 83236ea..25000d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,30 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") endif() endif() +# ── DebugView logging toggle ──────────────────────────────────── +# When ON, every ANS_DBG(...) call across the whole tree expands to an +# OutputDebugStringA() call visible in Sysinternals DebugView (Dbgview.exe). +# This is the single switch for verbose runtime diagnostics in ANSLPR, +# ANSCV (RTSP lifecycle, HW decoder auto-config), ANSODEngine (NV12 fast +# path, ORT/TRT engine selection), ANSFR (face recognizer state), etc. +# +# Enable it to diagnose field issues (e.g. "ALPR worked for a while then +# stopped"), then turn it back OFF for production because every ANS_DBG +# call adds a kernel round-trip and string formatting cost. +# +# Usage: +# cmake -B build -DANSCORE_DEBUGVIEW=ON # enable +# cmake -B build -DANSCORE_DEBUGVIEW=OFF # disable (default) +# +# Or toggle in CLion/VS: edit the cache variable ANSCORE_DEBUGVIEW. +option(ANSCORE_DEBUGVIEW "Enable ANS_DBG OutputDebugString logging for DebugView" OFF) +if(ANSCORE_DEBUGVIEW) + add_compile_definitions(ANSCORE_DEBUGVIEW=1) + message(STATUS "ANSCORE_DEBUGVIEW = ON — ANS_DBG verbose logging ENABLED (DebugView)") +else() + message(STATUS "ANSCORE_DEBUGVIEW = OFF — ANS_DBG verbose logging disabled (production)") +endif() + # ── External Dependencies ─────────────────────────────────────── include(cmake/Dependencies.cmake) diff --git a/core/ANSLicensingSystem/ANSLicense.h b/core/ANSLicensingSystem/ANSLicense.h index 7497409..b55fbf1 100644 --- a/core/ANSLicensingSystem/ANSLicense.h +++ b/core/ANSLicensingSystem/ANSLicense.h @@ -8,7 +8,7 @@ // Set to 0 for production builds to eliminate all debug output overhead. // ============================================================================ #ifndef ANSCORE_DEBUGVIEW -#define ANSCORE_DEBUGVIEW 0 // 1 = enabled (debug), 0 = disabled (production) +#define ANSCORE_DEBUGVIEW 1 // 1 = enabled (debug), 0 = disabled (production) #endif // ANS_DBG: Debug logging macro for DebugView (OutputDebugStringA on Windows). diff --git a/modules/ANSCV/ANSCVVendorGate.h b/modules/ANSCV/ANSCVVendorGate.h new file mode 100644 index 0000000..83f4792 --- /dev/null +++ b/modules/ANSCV/ANSCVVendorGate.h @@ -0,0 +1,59 @@ +#pragma once +// ANSCVVendorGate.h — Cached NVIDIA hardware check for ANSCV.dll. +// +// ANSCV.dll links against CUDA::cudart_static + CUDA::cublasLt + CUDA::nvjpeg +// because it hosts NVDEC hardware decode, NV12 GPU frame pool, and the RTSP / +// SRT / RTMP / MJPEG / FLV players that feed NV12 frames into the downstream +// inference DLLs (ANSLPR, ANSOCR, ANSFR). +// +// Several code paths in ANSCV call into the CUDA runtime unconditionally: +// • Post-NVDEC memory pool cleanup in Destroy/Reconnect +// • cudaGetDeviceCount() probes inside AutoConfigureHWDecoders +// • nvJPEG encoder helpers +// +// On NVIDIA hardware these are fine. On AMD / Intel / pure-CPU machines: +// • cudart_static is linked, but calling it wakes up CUDA driver state +// that was never needed — wastes address space and (when combined with +// DirectML decode on AMD) has been observed to destabilise amdkmdag. +// • The post-NVDEC cleanup runs even though no NVDEC decoder was ever +// created, which is pure waste on AMD/Intel. +// +// Solution: gate every CUDA runtime call behind this cached predicate, which +// evaluates CheckHardwareInformation() exactly once per process. If the +// detected engine is not NVIDIA_GPU, all CUDA/NVDEC cleanup paths become +// no-ops — decoders fall back to DXVA/D3D11VA/CPU automatically via the +// existing AutoConfigureHWDecoders_Platform() fallback. +// +// Mirrors the ANSLPR_OD / ANSOCR / ANSFR vendor gates that were added to +// ANSALPR_OD::LoadEngine, CreateANSOCRHandleEx, and CreateANSRFHandle. + +#include "ANSLicense.h" +#include + +namespace anscv_vendor_gate { + +// Lazily evaluates ANSLicenseHelper::CheckHardwareInformation() once and +// caches the result. Thread-safe: the first call on any thread performs +// the detection, all subsequent calls return the cached bool. Using an +// atomic bool + init-flag avoids pulling in std::call_once and its +// exception-safety overhead (the helper is on the hot decoder path). +[[nodiscard]] inline bool IsNvidiaGpuAvailable() noexcept { + static std::atomic s_state{0}; // 0 = unknown, 1 = NVIDIA, 2 = non-NVIDIA + int cached = s_state.load(std::memory_order_acquire); + if (cached != 0) return cached == 1; + try { + const ANSCENTER::EngineType detected = + ANSCENTER::ANSLicenseHelper::CheckHardwareInformation(); + const bool isNvidia = (detected == ANSCENTER::EngineType::NVIDIA_GPU); + // Last-writer-wins is fine — CheckHardwareInformation is deterministic. + s_state.store(isNvidia ? 1 : 2, std::memory_order_release); + return isNvidia; + } catch (...) { + // If detection throws (should not happen), fail safe to non-NVIDIA so + // we never activate CUDA runtime on unknown hardware. + s_state.store(2, std::memory_order_release); + return false; + } +} + +} // namespace anscv_vendor_gate diff --git a/modules/ANSCV/ANSFLV.cpp b/modules/ANSCV/ANSFLV.cpp index a861c3f..821a098 100644 --- a/modules/ANSCV/ANSFLV.cpp +++ b/modules/ANSCV/ANSFLV.cpp @@ -1,6 +1,7 @@ #include "ANSFLV.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include #include #include "media_codec.h" @@ -551,6 +552,12 @@ namespace ANSCENTER { #endif int ANSFLVClient::AutoConfigureHWDecoders(int maxPerGpuOverride) { + // Skip the CUDA probe on non-NVIDIA hardware — the Platform fallback + // handles Intel/AMD auto configuration. See ANSCVVendorGate.h. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + return AutoConfigureHWDecoders_Platform_FLV(); + } + int gpuCount = 0; cudaError_t err = cudaGetDeviceCount(&gpuCount); if (err != cudaSuccess || gpuCount <= 0) { diff --git a/modules/ANSCV/ANSMJPEG.cpp b/modules/ANSCV/ANSMJPEG.cpp index b320bae..56af91b 100644 --- a/modules/ANSCV/ANSMJPEG.cpp +++ b/modules/ANSCV/ANSMJPEG.cpp @@ -1,6 +1,7 @@ #include "ANSMJPEG.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include #include #include "media_codec.h" @@ -549,6 +550,12 @@ namespace ANSCENTER { #endif int ANSMJPEGClient::AutoConfigureHWDecoders(int maxPerGpuOverride) { + // Skip the CUDA probe on non-NVIDIA hardware — the Platform fallback + // handles Intel/AMD auto configuration. See ANSCVVendorGate.h. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + return AutoConfigureHWDecoders_Platform_MJPEG(); + } + int gpuCount = 0; cudaError_t err = cudaGetDeviceCount(&gpuCount); if (err != cudaSuccess || gpuCount <= 0) { diff --git a/modules/ANSCV/ANSRTMP.cpp b/modules/ANSCV/ANSRTMP.cpp index 6b8343c..dfe996e 100644 --- a/modules/ANSCV/ANSRTMP.cpp +++ b/modules/ANSCV/ANSRTMP.cpp @@ -1,6 +1,7 @@ #include "ANSRTMP.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include #include "media_codec.h" #include @@ -563,6 +564,12 @@ namespace ANSCENTER { #endif int ANSRTMPClient::AutoConfigureHWDecoders(int maxPerGpuOverride) { + // Skip the CUDA probe on non-NVIDIA hardware — the Platform fallback + // handles Intel/AMD auto configuration. See ANSCVVendorGate.h. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + return AutoConfigureHWDecoders_Platform_RTMP(); + } + int gpuCount = 0; cudaError_t err = cudaGetDeviceCount(&gpuCount); if (err != cudaSuccess || gpuCount <= 0) { diff --git a/modules/ANSCV/ANSRTSP.cpp b/modules/ANSCV/ANSRTSP.cpp index 1ed7a04..7536d5d 100644 --- a/modules/ANSCV/ANSRTSP.cpp +++ b/modules/ANSCV/ANSRTSP.cpp @@ -3,6 +3,7 @@ #include "ANSGpuFrameOps.h" #include "GpuNV12SlotPool.h" #include "ANSLicense.h" // ANS_DBG macro +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include #include #include @@ -136,17 +137,26 @@ namespace ANSCENTER { // memory → VRAM grows by ~200-300MB per destroy/create cycle. // cudaDeviceSynchronize ensures all pending GPU ops are done, then // cudaMemPool trim releases the freed blocks back to the OS. - cudaDeviceSynchronize(); - cudaMemPool_t memPool = nullptr; - int currentDev = 0; - cudaGetDevice(¤tDev); - if (cudaDeviceGetDefaultMemPool(&memPool, currentDev) == cudaSuccess && memPool) { - cudaMemPoolTrimTo(memPool, 0); // Release all unused memory + // + // AMD/Intel/CPU gate: this entire block is a no-op on non-NVIDIA + // machines because NVDEC never ran, the CUDA memory pool is empty, + // and calling cuda*() here would wake up cudart_static for nothing + // (and on AMD can destabilise amdkmdag when DirectML is active). + if (anscv_vendor_gate::IsNvidiaGpuAvailable()) { + cudaDeviceSynchronize(); + cudaMemPool_t memPool = nullptr; + int currentDev = 0; + cudaGetDevice(¤tDev); + if (cudaDeviceGetDefaultMemPool(&memPool, currentDev) == cudaSuccess && memPool) { + cudaMemPoolTrimTo(memPool, 0); // Release all unused memory + } + size_t vramFree = 0, vramTotal = 0; + cudaMemGetInfo(&vramFree, &vramTotal); + ANS_DBG("RTSP_Destroy", "NVDEC closed + memPool trimmed GPU%d VRAM=%zuMB/%zuMB", + currentDev, (vramTotal - vramFree) / (1024*1024), vramFree / (1024*1024)); + } else { + ANS_DBG("RTSP_Destroy", "non-NVIDIA hardware — skipped CUDA memory pool trim"); } - size_t vramFree = 0, vramTotal = 0; - cudaMemGetInfo(&vramFree, &vramTotal); - ANS_DBG("RTSP_Destroy", "NVDEC closed + memPool trimmed GPU%d VRAM=%zuMB/%zuMB", - currentDev, (vramTotal - vramFree) / (1024*1024), vramFree / (1024*1024)); } } static void VerifyGlobalANSRTSPLicense(const std::string& licenseKey) { @@ -281,23 +291,32 @@ namespace ANSCENTER { auto _rc1 = std::chrono::steady_clock::now(); // Force CUDA runtime to release cached memory from the destroyed NVDEC decoder. - cudaDeviceSynchronize(); - auto _rc2 = std::chrono::steady_clock::now(); - cudaMemPool_t memPool = nullptr; - int currentDev = 0; - cudaGetDevice(¤tDev); - if (cudaDeviceGetDefaultMemPool(&memPool, currentDev) == cudaSuccess && memPool) { - cudaMemPoolTrimTo(memPool, 0); - } - auto _rc3 = std::chrono::steady_clock::now(); - { - size_t vf = 0, vt = 0; - cudaMemGetInfo(&vf, &vt); + // Gated on NVIDIA: on AMD/Intel/CPU there was no NVDEC decoder and no + // CUDA memory pool to trim, so calling into cudart is pure overhead + // (and combined with DirectML on AMD has been observed to destabilise + // amdkmdag). See ANSCVVendorGate.h for the rationale. + if (anscv_vendor_gate::IsNvidiaGpuAvailable()) { + cudaDeviceSynchronize(); + auto _rc2 = std::chrono::steady_clock::now(); + cudaMemPool_t memPool = nullptr; + int currentDev = 0; + cudaGetDevice(¤tDev); + if (cudaDeviceGetDefaultMemPool(&memPool, currentDev) == cudaSuccess && memPool) { + cudaMemPoolTrimTo(memPool, 0); + } + auto _rc3 = std::chrono::steady_clock::now(); + { + size_t vf = 0, vt = 0; + cudaMemGetInfo(&vf, &vt); + double closeMs = std::chrono::duration(_rc1 - _rc0).count(); + double syncMs = std::chrono::duration(_rc2 - _rc1).count(); + double trimMs = std::chrono::duration(_rc3 - _rc2).count(); + ANS_DBG("RTSP_Reconnect", "close=%.1fms sync=%.1fms trim=%.1fms VRAM=%zuMB/%zuMB", + closeMs, syncMs, trimMs, (vt - vf) / (1024*1024), vf / (1024*1024)); + } + } else { double closeMs = std::chrono::duration(_rc1 - _rc0).count(); - double syncMs = std::chrono::duration(_rc2 - _rc1).count(); - double trimMs = std::chrono::duration(_rc3 - _rc2).count(); - ANS_DBG("RTSP_Reconnect", "close=%.1fms sync=%.1fms trim=%.1fms VRAM=%zuMB/%zuMB", - closeMs, syncMs, trimMs, (vt - vf) / (1024*1024), vf / (1024*1024)); + ANS_DBG("RTSP_Reconnect", "close=%.1fms (non-NVIDIA — CUDA memory pool trim skipped)", closeMs); } RTSP_DBG("[Reconnect] AFTER close() this=%p", (void*)this); @@ -882,6 +901,14 @@ namespace ANSCENTER { #endif int ANSRTSPClient::AutoConfigureHWDecoders(int maxPerGpuOverride) { + // Skip the CUDA probe entirely on non-NVIDIA hardware — the Platform + // fallback (DXGI on Windows, sysfs on Linux) handles Intel/AMD auto + // configuration, and calling cudaGetDeviceCount() on AMD wakes up + // cudart_static for no benefit. See ANSCVVendorGate.h. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + return AutoConfigureHWDecoders_Platform(); + } + int gpuCount = 0; cudaError_t err = cudaGetDeviceCount(&gpuCount); if (err != cudaSuccess || gpuCount <= 0) { diff --git a/modules/ANSCV/ANSSRT.cpp b/modules/ANSCV/ANSSRT.cpp index f003312..b25d809 100644 --- a/modules/ANSCV/ANSSRT.cpp +++ b/modules/ANSCV/ANSSRT.cpp @@ -1,6 +1,7 @@ #include "ANSSRT.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameOps.h" +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include #include "media_codec.h" #include @@ -577,6 +578,13 @@ namespace ANSCENTER { #endif int ANSSRTClient::AutoConfigureHWDecoders(int maxPerGpuOverride) { + // Skip the CUDA probe on non-NVIDIA hardware — the Platform fallback + // (DXGI/sysfs) handles Intel/AMD auto configuration. See + // ANSCVVendorGate.h for rationale. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + return AutoConfigureHWDecoders_Platform_SRT(); + } + int gpuCount = 0; cudaError_t err = cudaGetDeviceCount(&gpuCount); if (err != cudaSuccess || gpuCount <= 0) { diff --git a/modules/ANSCV/ANSWebcam.cpp b/modules/ANSCV/ANSWebcam.cpp index e951314..0ea86d5 100644 --- a/modules/ANSCV/ANSWebcam.cpp +++ b/modules/ANSCV/ANSWebcam.cpp @@ -1,6 +1,7 @@ #include "ANSWEBCAM.h" #include "ANSMatRegistry.h" #include "ANSGpuFrameRegistry.h" +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() #include #include extern "C" { @@ -914,6 +915,15 @@ namespace ANSCENTER { return result; } void ANSWEBCAMPlayer::uploadPlanarBGRToGPU(const cv::Mat& inputMat, unsigned char** data) { + // Refuse on non-NVIDIA — cudaMalloc/cudaMemcpy are NVIDIA-only. + // The public entry point encodeMatToJpegWithNvJPEG() also guards, + // but defense-in-depth in case a future caller wires this up directly. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + this->_logger.LogWarn("ANSWEBCAMPlayer::uploadPlanarBGRToGPU", + "skipped — non-NVIDIA hardware, nvJPEG path unavailable", __FILE__, __LINE__); + if (data) *data = nullptr; + return; + } std::lock_guard lock(_mutex); try { int width = inputMat.cols; @@ -933,6 +943,13 @@ namespace ANSCENTER { } std::string ANSWEBCAMPlayer::encodeMatToJpegWithNvJPEG(const cv::Mat& inputMat, int quality) { + // nvJPEG encoder is NVIDIA-only (part of CUDA toolkit). Refuse on + // AMD/Intel/CPU and let the caller fall back to the turbojpeg path. + if (!anscv_vendor_gate::IsNvidiaGpuAvailable()) { + this->_logger.LogWarn("ANSWEBCAMPlayer::encodeMatToJpegWithNvJPEG", + "nvJPEG requires NVIDIA GPU; falling back to last cached JPEG", __FILE__, __LINE__); + return _lastJpegImage; + } std::lock_guard lock(_mutex); try { // Image dimensions diff --git a/modules/ANSCV/dllmain.cpp b/modules/ANSCV/dllmain.cpp index 8a0dbe1..c5dd239 100644 --- a/modules/ANSCV/dllmain.cpp +++ b/modules/ANSCV/dllmain.cpp @@ -1,5 +1,7 @@ // dllmain.cpp : Defines the entry point for the DLL application. #include "pch.h" +#include "ANSCVVendorGate.h" // anscv_vendor_gate::IsNvidiaGpuAvailable() +#include "ANSLicense.h" // ANSCENTER::EngineType, CheckHardwareInformation #include #include #include @@ -61,6 +63,15 @@ BOOL APIENTRY DllMain( HMODULE hModule, // Pinning keeps the code pages mapped; the OS kills all threads when // the process exits, so this is safe and is Microsoft's recommended // pattern for DLLs that own threads. + // + // CRITICAL: do NOT call CheckHardwareInformation() or + // anscv_vendor_gate::IsNvidiaGpuAvailable() here. DllMain holds the + // OS loader lock (LdrpLoaderLock). CheckHardwareInformation touches + // hwinfo → DXGI / WMI / COM which internally call LoadLibrary; doing + // that while holding the loader lock causes a classic loader-lock + // deadlock (confirmed by stress-test hang). The vendor gate will + // lazy-initialise on the first real call from worker code, which + // runs with the loader lock released. { HMODULE hSelf = nullptr; GetModuleHandleExW( diff --git a/modules/ANSFR/ANSFaceRecognizer.cpp b/modules/ANSFR/ANSFaceRecognizer.cpp index 007fbe2..012af8d 100644 --- a/modules/ANSFR/ANSFaceRecognizer.cpp +++ b/modules/ANSFR/ANSFaceRecognizer.cpp @@ -680,6 +680,19 @@ namespace ANSCENTER { std::vector ANSFaceRecognizer::RunArcFace(const cv::Mat& inputImage) { std::vector embedding; + // Defense-in-depth: this function uses m_gpuStream / cv::cuda::GpuMat + // upload path, which is only valid on NVIDIA hardware. Callers in + // Feature() and ExtractEmbeddings() already gate on engineType, but + // the method is public — refuse to run on AMD/Intel/CPU so we never + // touch m_gpuStream (lazy-initialized, nullptr on non-NVIDIA) or + // m_gpuRgb.upload() which would activate the CUDA runtime. + if (engineType != EngineType::NVIDIA_GPU) { + _logger.LogError("ANSFaceRecognizer::RunArcFace", + "RunArcFace is NVIDIA-only; called on engineType=" + + std::to_string(static_cast(engineType)), __FILE__, __LINE__); + return embedding; + } + // Early validation before locking if (inputImage.empty()) { _logger.LogError("ANSFaceRecognizer::RunArcFace", @@ -701,6 +714,13 @@ namespace ANSCENTER { return embedding; } + if (!m_gpuStream || !m_trtEngine) { + _logger.LogError("ANSFaceRecognizer::RunArcFace", + "GPU stream or TRT engine not available (engineType=" + + std::to_string(static_cast(engineType)) + ")", __FILE__, __LINE__); + return embedding; + } + try { // CPU preprocessing: resize + BGR→RGB before GPU upload // Reduces PCIe transfer and eliminates GPU cvtColor/resize overhead @@ -761,6 +781,17 @@ namespace ANSCENTER { { std::vector> embeddings; + // Defense-in-depth: TensorRT + cv::cuda::GpuMat batch path is NVIDIA-only. + // Callers in ExtractEmbeddings() already gate on engineType, but this is a + // public method — refuse to run on AMD/Intel/CPU so we never touch the + // TRT engine or cv::cuda primitives on non-NVIDIA hardware. + if (engineType != EngineType::NVIDIA_GPU) { + _logger.LogError("ANSFaceRecognizer::RunArcFaceBatch", + "RunArcFaceBatch is NVIDIA-only; called on engineType=" + + std::to_string(static_cast(engineType)), __FILE__, __LINE__); + return embeddings; + } + try { // Early validation checks if (!_isInitialized) { @@ -775,6 +806,12 @@ namespace ANSCENTER { return embeddings; } + if (!m_gpuStream) { + _logger.LogError("ANSFaceRecognizer::RunArcFaceBatch", + "GPU stream not initialized", __FILE__, __LINE__); + return embeddings; + } + if (faceROIs.empty()) { return embeddings; } diff --git a/modules/ANSFR/dllmain.cpp b/modules/ANSFR/dllmain.cpp index 3a9709f..9689b8d 100644 --- a/modules/ANSFR/dllmain.cpp +++ b/modules/ANSFR/dllmain.cpp @@ -97,14 +97,33 @@ public: }; // Determine maxSlotsPerGpu based on GPU topology: -// 1 GPU → 1 (single slot, no round-robin needed) -// >1 GPU, VRAM<24GB → 1 (round-robin: 1 slot per GPU) -// >1 GPU, VRAM≥24GB → -1 (elastic: on-demand slot growth) +// non-NVIDIA (AMD/Intel/CPU) → 1 (no TensorRT pool, never grows) +// 1 NVIDIA GPU → 1 (single slot, no round-robin needed) +// >1 GPU, VRAM<24GB → 1 (round-robin: 1 slot per GPU) +// >1 GPU, VRAM≥24GB → -1 (elastic: on-demand slot growth) +// +// IMPORTANT: Must be gated on CheckHardwareInformation() first — calling +// cudaGetDeviceCount/cudaSetDevice/cudaMemGetInfo on non-NVIDIA hardware +// wakes up the CUDA runtime unnecessarily and, combined with DirectML on +// AMD, has been observed to trigger amdkmdag instability. Return 1 early +// on anything that isn't a detected NVIDIA GPU so the TRT pool is never +// exercised on those machines. static int GetPoolMaxSlotsPerGpu() { static int s_result = INT_MIN; static std::mutex s_mutex; std::lock_guard lk(s_mutex); if (s_result != INT_MIN) return s_result; + + const ANSCENTER::EngineType detected = + ANSCENTER::ANSLicenseHelper::CheckHardwareInformation(); + if (detected != ANSCENTER::EngineType::NVIDIA_GPU) { + s_result = 1; + std::cout << "Info [FR GPU]: engineType=" << static_cast(detected) + << " — not NVIDIA, TRT pool disabled (slot=1), skipping CUDA probe" + << std::endl; + return s_result; + } + int gpuCount = 0; cudaGetDeviceCount(&gpuCount); if (gpuCount <= 1) { @@ -211,6 +230,26 @@ extern "C" ANSFR_API int CreateANSRFHandle(ANSCENTER::ANSFacialRecognition** if (!Handle || !licenseKey || !configFilePath || !databaseFilePath || !recogniserFilePath) return -1; + // Log the detected vendor path so field triage between NVIDIA / AMD / + // Intel / CPU machines is trivial from the debug log. Mirrors the + // vendorTag logging already in ANSLPR_OD::LoadEngine and ANSOCR + // CreateANSOCRHandleEx. + { + ANSCENTER::EngineType detected = + ANSCENTER::ANSLicenseHelper::CheckHardwareInformation(); + const char* vendorTag = + detected == ANSCENTER::EngineType::NVIDIA_GPU ? "NVIDIA_GPU (TensorRT + CUDA preproc, SCRFD face detector)" : + detected == ANSCENTER::EngineType::AMD_GPU ? "AMD_GPU (ONNX Runtime / DirectML, OV face detector, NV12/CUDA DISABLED)" : + detected == ANSCENTER::EngineType::OPENVINO_GPU ? "OPENVINO_GPU (OpenVINO, OV face detector, NV12/CUDA DISABLED)" : + "CPU (ONNX Runtime / OpenVINO CPU, NV12/CUDA DISABLED)"; + char buf[224]; + snprintf(buf, sizeof(buf), + "[ANSFR] CreateANSRFHandle: detected engineType=%d [%s]\n", + static_cast(detected), vendorTag); + OutputDebugStringA(buf); + std::cout << buf; + } + // Release existing handle if called twice (prevents leak from LabVIEW) if (*Handle) { if (UnregisterFRHandle(*Handle)) { diff --git a/modules/ANSLPR/ANSALPR_OV.cpp b/modules/ANSLPR/ANSALPR_OV.cpp deleted file mode 100644 index af06275..0000000 --- a/modules/ANSLPR/ANSALPR_OV.cpp +++ /dev/null @@ -1,843 +0,0 @@ -#include "ANSLPR_OV.h" -namespace ANSCENTER { - - void tryPush(const std::weak_ptr& worker, std::shared_ptr&& task) { - try { - std::shared_ptr(worker)->push(task); - } catch (const std::bad_weak_ptr&) {} - } - - void fillROIColor(cv::Mat& displayImage, cv::Rect roi, cv::Scalar color, double opacity) { - if (opacity > 0) { - roi = roi & cv::Rect(0, 0, displayImage.cols, displayImage.rows); - cv::Mat textROI = displayImage(roi); - cv::addWeighted(color, opacity, textROI, 1.0 - opacity , 0.0, textROI); - } - } - - void putTextOnImage(cv::Mat& displayImage, std::string str, cv::Point p, - cv::HersheyFonts font, double fontScale, cv::Scalar color, - int thickness = 1, cv::Scalar bgcolor = cv::Scalar(), - double opacity = 0) { - int baseline = 0; - cv::Size textSize = cv::getTextSize(str, font, 0.5, 1, &baseline); - fillROIColor(displayImage, cv::Rect(cv::Point(p.x, p.y + baseline), - cv::Point(p.x + textSize.width, p.y - textSize.height)), - bgcolor, opacity); - cv::putText(displayImage, str, p, font, fontScale, color, thickness); - } - - Detector::Detector(ov::Core& core, - const std::string& deviceName, - const std::string& xmlPath, - const std::vector& detectionTresholds, - const bool autoResize) : - m_autoResize(autoResize), - m_detectionTresholds{ detectionTresholds } - { - slog::info << "Reading model: " << xmlPath << slog::endl; - std::shared_ptr model = core.read_model(xmlPath); - logBasicModelInfo(model); - // Check model inputs and outputs - ov::OutputVector inputs = model->inputs(); - if (inputs.size() != 1) { - throw std::logic_error("Detector should have only one input"); - } - - m_detectorInputName = model->input().get_any_name(); - - ov::Layout modelLayout = ov::layout::get_layout(model->input()); - if (modelLayout.empty()) - modelLayout = { "NCHW" }; - - ov::OutputVector outputs = model->outputs(); - if (outputs.size() != 1) { - throw std::logic_error("Vehicle Detection network should have only one output"); - } - - ov::Output output = outputs[0]; - - m_detectorOutputName = output.get_any_name(); - ov::Shape output_shape = output.get_shape(); - - if (output_shape.size() != 4) { - throw std::logic_error("Incorrect output dimensions for SSD"); - } - - if (maxProposalCount != output_shape[2]) { - throw std::logic_error("unexpected ProposalCount"); - } - if (objectSize != output_shape[3]) { - throw std::logic_error("Output should have 7 as a last dimension"); - } - - ov::preprocess::PrePostProcessor ppp(model); - ov::preprocess::InputInfo& inputInfo = ppp.input(); - ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor(); - // configure desired input type and layout, the - // use preprocessor to convert to actual model input type and layout - inputTensorInfo.set_element_type(ov::element::u8); - inputTensorInfo.set_layout({ "NHWC" }); - if (autoResize) { - inputTensorInfo.set_spatial_dynamic_shape(); - } - - ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model(); - inputModelInfo.set_layout(modelLayout); - - ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess(); - preProcessSteps.convert_layout(modelLayout); - preProcessSteps.convert_element_type(ov::element::f32); - if (autoResize) { - preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); - } - - model = ppp.build(); - - slog::info << "Preprocessor configuration: " << slog::endl; - slog::info << ppp << slog::endl; - m_compiled_model = core.compile_model(model, deviceName); - logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "Vehicle And License Plate Detection"); - } - - ov::InferRequest Detector::createInferRequest() { - return m_compiled_model.create_infer_request(); - } - void Detector::setImage(ov::InferRequest& inferRequest, const cv::Mat& img) { - ov::Tensor inputTensor = inferRequest.get_tensor(m_detectorInputName); - ov::Shape shape = inputTensor.get_shape(); - if (m_autoResize) { - if (!img.isSubmatrix()) { - // just wrap Mat object with Tensor without additional memory allocation - ov::Tensor frameTensor = wrapMat2Tensor(img); - inferRequest.set_tensor(m_detectorInputName, frameTensor); - } - else { - throw std::logic_error("Sparse matrix are not supported"); - } - } - else { - // resize and copy data from image to tensor using OpenCV - resize2tensor(img, inputTensor); - } - } - std::list Detector::getResults(ov::InferRequest& inferRequest, - cv::Size upscale, - std::vector& rawResults) { - // there is no big difference if InferReq of detector from another device is passed - // because the processing is the same for the same topology - std::list results; - ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName); - const float* const detections = output_tensor.data(); - // pretty much regular SSD post-processing - for (int i = 0; i < maxProposalCount; i++) { - float image_id = detections[i * objectSize + 0]; // in case of batch - if (image_id < 0) { // indicates end of detections - break; - } - size_t label = static_cast(detections[i * objectSize + 1]); - float confidence = detections[i * objectSize + 2]; - if (label - 1 < m_detectionTresholds.size() && confidence < m_detectionTresholds[label - 1]) { - continue; - } - - cv::Rect rect; - rect.x = static_cast(detections[i * objectSize + 3] * upscale.width); - rect.y = static_cast(detections[i * objectSize + 4] * upscale.height); - rect.width = static_cast(detections[i * objectSize + 5] * upscale.width) - rect.x; - rect.height = static_cast(detections[i * objectSize + 6] * upscale.height) - rect.y; - results.push_back(Result{ label, confidence, rect }); - std::ostringstream rawResultsStream; - rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence - << " (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")"; - rawResults.push_back(rawResultsStream.str()); - } - return results; - } - - VehicleAttributesClassifier::VehicleAttributesClassifier(ov::Core& core, - const std::string& deviceName, - const std::string& xmlPath, - const bool autoResize) :m_autoResize(autoResize) - { - slog::info << "Reading model: " << xmlPath << slog::endl; - std::shared_ptr model = core.read_model(xmlPath); - logBasicModelInfo(model); - ov::OutputVector inputs = model->inputs(); - if (inputs.size() != 1) { - throw std::logic_error("Vehicle Attribs topology should have only one input"); - } - m_attributesInputName = model->input().get_any_name(); - ov::Layout modelLayout = ov::layout::get_layout(model->input()); - if (modelLayout.empty()) - modelLayout = { "NCHW" }; - - ov::OutputVector outputs = model->outputs(); - if (outputs.size() != 2) { - throw std::logic_error("Vehicle Attribs Network expects networks having two outputs"); - } - - // color is the first output - m_outputNameForColor = outputs[0].get_any_name(); - // type is the second output. - m_outputNameForType = outputs[1].get_any_name(); - ov::preprocess::PrePostProcessor ppp(model); - ov::preprocess::InputInfo& inputInfo = ppp.input(); - ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor(); - // configure desired input type and layout, the - // use preprocessor to convert to actual model input type and layout - inputTensorInfo.set_element_type(ov::element::u8); - inputTensorInfo.set_layout({ "NHWC" }); - if (autoResize) { - inputTensorInfo.set_spatial_dynamic_shape(); - } - ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess(); - preProcessSteps.convert_layout(modelLayout); - preProcessSteps.convert_element_type(ov::element::f32); - if (autoResize) { - preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); - } - - ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model(); - inputModelInfo.set_layout(modelLayout); - - model = ppp.build(); - - slog::info << "Preprocessor configuration: " << slog::endl; - slog::info << ppp << slog::endl; - - m_compiled_model = core.compile_model(model, deviceName); - logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "Vehicle Attributes Recognition"); - } - ov::InferRequest VehicleAttributesClassifier::createInferRequest() { - return m_compiled_model.create_infer_request(); - } - void VehicleAttributesClassifier::setImage(ov::InferRequest& inferRequest, - const cv::Mat& img, - const cv::Rect vehicleRect) - { - ov::Tensor inputTensor = inferRequest.get_tensor(m_attributesInputName); - ov::Shape shape = inputTensor.get_shape(); - if (m_autoResize) { - ov::Tensor frameTensor = wrapMat2Tensor(img); - ov::Coordinate p00({ 0, static_cast(vehicleRect.y), static_cast(vehicleRect.x), 0 }); - ov::Coordinate p01({ 1, static_cast(vehicleRect.y + vehicleRect.height), static_cast(vehicleRect.x) + vehicleRect.width, 3 }); - ov::Tensor roiTensor(frameTensor, p00, p01); - inferRequest.set_tensor(m_attributesInputName, roiTensor); - } - else { - const cv::Mat& vehicleImage = img(vehicleRect); - resize2tensor(vehicleImage, inputTensor); - } - } - std::pair VehicleAttributesClassifier::getResults(ov::InferRequest& inferRequest) { - static const std::string colors[] = { - "white", "gray", "yellow", "red", "green", "blue", "black" - }; - static const std::string types[] = { - "car", "van", "truck", "bus" - }; - - // 7 possible colors for each vehicle and we should select the one with the maximum probability - ov::Tensor colorsTensor = inferRequest.get_tensor(m_outputNameForColor); - const float* colorsValues = colorsTensor.data(); - - // 4 possible types for each vehicle and we should select the one with the maximum probability - ov::Tensor typesTensor = inferRequest.get_tensor(m_outputNameForType); - const float* typesValues = typesTensor.data(); - - const auto color_id = std::max_element(colorsValues, colorsValues + 7) - colorsValues; - const auto type_id = std::max_element(typesValues, typesValues + 4) - typesValues; - - return std::pair(colors[color_id], types[type_id]); - } - - Lpr::Lpr(ov::Core& core, - const std::string& deviceName, - const std::string& xmlPath, - const bool autoResize) :m_autoResize(autoResize) - { - slog::info << "Reading model: " << xmlPath << slog::endl; - std::shared_ptr model = core.read_model(xmlPath); - logBasicModelInfo(model); - // LPR network should have 2 inputs (and second is just a stub) and one output - // Check inputs - ov::OutputVector inputs = model->inputs(); - if (inputs.size() != 1 && inputs.size() != 2) { - throw std::logic_error("LPR should have 1 or 2 inputs"); - } - - for (auto input : inputs) { - if (input.get_shape().size() == 4) { - m_LprInputName = input.get_any_name(); - m_modelLayout = ov::layout::get_layout(input); - if (m_modelLayout.empty()) - m_modelLayout = { "NCHW" }; - } - // LPR model that converted from Caffe have second a stub input - if (input.get_shape().size() == 2) - m_LprInputSeqName = input.get_any_name(); - } - - // Check outputs - - m_maxSequenceSizePerPlate = 1; - - ov::OutputVector outputs = model->outputs(); - if (outputs.size() != 1) { - throw std::logic_error("LPR should have 1 output"); - } - - m_LprOutputName = outputs[0].get_any_name(); - - for (size_t dim : outputs[0].get_shape()) { - if (dim == 1) { - continue; - } - if (m_maxSequenceSizePerPlate == 1) { - m_maxSequenceSizePerPlate = dim; - } - else { - throw std::logic_error("Every dimension of LPR output except for one must be of size 1"); - } - } - - ov::preprocess::PrePostProcessor ppp(model); - ov::preprocess::InputInfo& inputInfo = ppp.input(m_LprInputName); - ov::preprocess::InputTensorInfo& inputTensorInfo = inputInfo.tensor(); - // configure desired input type and layout, the - // use preprocessor to convert to actual model input type and layout - inputTensorInfo.set_element_type(ov::element::u8); - inputTensorInfo.set_layout({ "NHWC" }); - if (autoResize) { - inputTensorInfo.set_spatial_dynamic_shape(); - } - - ov::preprocess::PreProcessSteps& preProcessSteps = inputInfo.preprocess(); - preProcessSteps.convert_layout(m_modelLayout); - preProcessSteps.convert_element_type(ov::element::f32); - if (autoResize) { - preProcessSteps.resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); - } - - ov::preprocess::InputModelInfo& inputModelInfo = inputInfo.model(); - inputModelInfo.set_layout(m_modelLayout); - - model = ppp.build(); - - slog::info << "Preprocessor configuration: " << slog::endl; - slog::info << ppp << slog::endl; - - m_compiled_model = core.compile_model(model, deviceName); - logCompiledModelInfo(m_compiled_model, xmlPath, deviceName, "License Plate Recognition"); - } - ov::InferRequest Lpr::createInferRequest() { - return m_compiled_model.create_infer_request(); - } - void Lpr::setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect plateRect) { - ov::Tensor inputTensor = inferRequest.get_tensor(m_LprInputName); - ov::Shape shape = inputTensor.get_shape(); - if ((shape.size() == 4) && m_autoResize) { - // autoResize is set - ov::Tensor frameTensor = wrapMat2Tensor(img); - ov::Coordinate p00({ 0, static_cast(plateRect.y), static_cast(plateRect.x), 0 }); - ov::Coordinate p01({ 1, static_cast(plateRect.y + plateRect.height), static_cast(plateRect.x + plateRect.width), 3 }); - ov::Tensor roiTensor(frameTensor, p00, p01); - inferRequest.set_tensor(m_LprInputName, roiTensor); - } - else { - const cv::Mat& vehicleImage = img(plateRect); - resize2tensor(vehicleImage, inputTensor); - } - - if (m_LprInputSeqName != "") { - ov::Tensor inputSeqTensor = inferRequest.get_tensor(m_LprInputSeqName); - float* data = inputSeqTensor.data(); - std::fill(data, data + inputSeqTensor.get_shape()[0], 1.0f); - } - } - std::string Lpr::getResults(ov::InferRequest& inferRequest) { - static const char* const items[] = { - "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", - "", "", "", "", - "", "", "", "", - "", "", "", "", - "", "", "", "", - "", "", "", "", - "", "", "", "", - "", "", "", "", - "", "", "", "", - "", "", - "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", - "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", - "U", "V", "W", "X", "Y", "Z" - }; - std::string result; - result.reserve(14u + 6u); // the longest province name + 6 plate signs - - ov::Tensor lprOutputTensor = inferRequest.get_tensor(m_LprOutputName); - ov::element::Type precision = lprOutputTensor.get_element_type(); - - // up to 88 items per license plate, ended with "-1" - switch (precision) { - case ov::element::i32: - { - const auto data = lprOutputTensor.data(); - for (int i = 0; i < m_maxSequenceSizePerPlate; i++) { - int32_t val = data[i]; - if (val == -1) { - break; - } - result += items[val]; - } - } - break; - - case ov::element::f32: - { - const auto data = lprOutputTensor.data(); - for (int i = 0; i < m_maxSequenceSizePerPlate; i++) { - int32_t val = int32_t(data[i]); - if (val == -1) { - break; - } - result += items[val]; - } - } - break; - - default: - throw std::logic_error("Not expected output blob precision"); - break; - } - return result; - } - - // Utilities - ReborningVideoFrame::~ReborningVideoFrame() { - try { - const std::shared_ptr& worker = std::shared_ptr(context.readersContext.readersWorker); - context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock(); - const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID]; - context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock(); - std::shared_ptr reborn = std::make_shared(context, sourceID, frameId, frame); - worker->push(std::make_shared(reborn)); - } - catch (const std::bad_weak_ptr&) {} - } - - void ResAggregator::process() { - Context& context = static_cast(sharedVideoFrame.get())->context; - context.freeDetectionInfersCount += context.detectorsInfers.inferRequests.lockedSize(); - context.frameCounter++; - context.boxesAndDescrs = boxesAndDescrs; - try { - std::shared_ptr(context.resAggregatorsWorker)->stop(); - } - catch (const std::bad_weak_ptr&) {} - } - - bool DetectionsProcessor::isReady() { - Context& context = static_cast(sharedVideoFrame.get())->context; - if (requireGettingNumberOfDetections) { - classifiersAggregator = std::make_shared(sharedVideoFrame); - std::list results; - results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections); - for (Detector::Result result : results) { - switch (result.label) { - case 1:// Vehicle - { - vehicleRects.emplace_back(result.location & cv::Rect{ cv::Point(0, 0), sharedVideoFrame->frame.size() }); - break; - } - case 2:// License Plate - { - // expanding a bounding box a bit, better for the license plate recognition - result.location.x -= 5; - result.location.y -= 5; - result.location.width += 10; - result.location.height += 10; - plateRects.emplace_back(result.location & cv::Rect{ cv::Point(0, 0), sharedVideoFrame->frame.size() }); - break; - } - default: - throw std::runtime_error("Unexpected detection results"); // must never happen - break; - } - } - context.detectorsInfers.inferRequests.lockedPushBack(*inferRequest); - requireGettingNumberOfDetections = false; - } - - if ((vehicleRects.empty()) && (plateRects.empty())) { - return true; - } - else { - InferRequestsContainer& attributesInfers = context.attributesInfers; - attributesInfers.inferRequests.mutex.lock(); - const std::size_t numberOfAttributesInferRequestsAcquired = std::min(vehicleRects.size(), attributesInfers.inferRequests.container.size()); - reservedAttributesRequests.assign(attributesInfers.inferRequests.container.end() - numberOfAttributesInferRequestsAcquired,attributesInfers.inferRequests.container.end()); - attributesInfers.inferRequests.container.erase(attributesInfers.inferRequests.container.end() - numberOfAttributesInferRequestsAcquired,attributesInfers.inferRequests.container.end()); - attributesInfers.inferRequests.mutex.unlock(); - - InferRequestsContainer& platesInfers = context.platesInfers; - platesInfers.inferRequests.mutex.lock(); - const std::size_t numberOfLprInferRequestsAcquired = std::min(plateRects.size(), platesInfers.inferRequests.container.size()); - reservedLprRequests.assign(platesInfers.inferRequests.container.end() - numberOfLprInferRequestsAcquired, platesInfers.inferRequests.container.end()); - platesInfers.inferRequests.container.erase(platesInfers.inferRequests.container.end() - numberOfLprInferRequestsAcquired,platesInfers.inferRequests.container.end()); - platesInfers.inferRequests.mutex.unlock(); - return numberOfAttributesInferRequestsAcquired || numberOfLprInferRequestsAcquired; - } - } - - void DetectionsProcessor::process() { - Context& context = static_cast(sharedVideoFrame.get())->context; - auto vehicleRectsIt = vehicleRects.begin(); - for (auto attributesRequestIt = reservedAttributesRequests.begin(); attributesRequestIt != reservedAttributesRequests.end(); - vehicleRectsIt++, attributesRequestIt++) { - const cv::Rect vehicleRect = *vehicleRectsIt; - ov::InferRequest& attributesRequest = *attributesRequestIt; - context.detectionsProcessorsContext.vehicleAttributesClassifier.setImage(attributesRequest, sharedVideoFrame->frame, vehicleRect); - - attributesRequest.set_callback( - std::bind( - [](std::shared_ptr classifiersAggregator, - ov::InferRequest& attributesRequest, - cv::Rect rect, - Context& context) { - attributesRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object - const std::pair& attributes =context.detectionsProcessorsContext.vehicleAttributesClassifier.getResults(attributesRequest); - if (((classifiersAggregator->sharedVideoFrame->frameId == 0 && !context.isVideo) || context.isVideo)) { - classifiersAggregator->rawAttributes.lockedPushBack("Vehicle Attributes results:" + attributes.first + ';' + attributes.second); - } - classifiersAggregator->push(BboxAndDescr{ BboxAndDescr::ObjectType::VEHICLE, rect, attributes.first + ' ' + attributes.second }); - context.attributesInfers.inferRequests.lockedPushBack(attributesRequest); - }, - classifiersAggregator, - std::ref(attributesRequest), - vehicleRect, - std::ref(context))); - attributesRequest.start_async(); - } - vehicleRects.erase(vehicleRects.begin(), vehicleRectsIt); - - auto plateRectsIt = plateRects.begin(); - for (auto lprRequestsIt = reservedLprRequests.begin(); lprRequestsIt != reservedLprRequests.end(); plateRectsIt++, lprRequestsIt++) { - const cv::Rect plateRect = *plateRectsIt; - ov::InferRequest& lprRequest = *lprRequestsIt; - context.detectionsProcessorsContext.lpr.setImage(lprRequest, sharedVideoFrame->frame, plateRect); - - lprRequest.set_callback( - std::bind( - [](std::shared_ptr classifiersAggregator, - ov::InferRequest& lprRequest, - cv::Rect rect, - Context& context) { - lprRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object - - std::string result = context.detectionsProcessorsContext.lpr.getResults(lprRequest); - - if (((classifiersAggregator->sharedVideoFrame->frameId == 0 && !context.isVideo) || context.isVideo)) { - classifiersAggregator->rawDecodedPlates.lockedPushBack("License Plate Recognition results:" + result); - } - classifiersAggregator->push(BboxAndDescr{ BboxAndDescr::ObjectType::PLATE, rect, std::move(result) }); - context.platesInfers.inferRequests.lockedPushBack(lprRequest); - }, classifiersAggregator, - std::ref(lprRequest), - plateRect, - std::ref(context))); - - lprRequest.start_async(); - } - plateRects.erase(plateRects.begin(), plateRectsIt); - - if (!vehicleRects.empty() || !plateRects.empty()) { - tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker, - std::make_shared(sharedVideoFrame, std::move(classifiersAggregator), std::move(vehicleRects), std::move(plateRects))); - } - } - - bool InferTask::isReady() { - InferRequestsContainer& detectorsInfers = static_cast(sharedVideoFrame.get())->context.detectorsInfers; - if (detectorsInfers.inferRequests.container.empty()) { - return false; - } - else { - detectorsInfers.inferRequests.mutex.lock(); - if (detectorsInfers.inferRequests.container.empty()) { - detectorsInfers.inferRequests.mutex.unlock(); - return false; - } - else { - return true; // process() will unlock the mutex - } - } - } - - void InferTask::process() { - Context& context = static_cast(sharedVideoFrame.get())->context; - InferRequestsContainer& detectorsInfers = context.detectorsInfers; - std::reference_wrapper inferRequest = detectorsInfers.inferRequests.container.back(); - detectorsInfers.inferRequests.container.pop_back(); - detectorsInfers.inferRequests.mutex.unlock(); - context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame); - inferRequest.get().set_callback( - std::bind( - [](VideoFrame::Ptr sharedVideoFrame, - ov::InferRequest& inferRequest, - Context& context) { - inferRequest.set_callback([](std::exception_ptr) {}); // destroy the stored bind object - tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker, - std::make_shared(sharedVideoFrame, &inferRequest)); - }, sharedVideoFrame, - inferRequest, - std::ref(context))); - - inferRequest.get().start_async(); - // do not push as callback does it - } - - bool Reader::isReady() { - Context& context = static_cast(sharedVideoFrame.get())->context; - context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].lock(); - if (context.readersContext.lastCapturedFrameIds[sharedVideoFrame->sourceID] + 1 == sharedVideoFrame->frameId) { - return true; - } - else { - context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].unlock(); - return false; - } - } - void Reader::process() { - unsigned sourceID = sharedVideoFrame->sourceID; - sharedVideoFrame->timestamp = std::chrono::steady_clock::now(); - Context& context = static_cast(sharedVideoFrame.get())->context; - const std::vector>& inputChannels = context.readersContext.inputChannels; - if (inputChannels[sourceID]->read(sharedVideoFrame->frame)) { - context.readersContext.lastCapturedFrameIds[sourceID]++; - context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock(); - tryPush(context.inferTasksContext.inferTasksWorker, std::make_shared(sharedVideoFrame)); - } - else { - context.readersContext.lastCapturedFrameIds[sourceID]++; - context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock(); - try { - std::shared_ptr(context.resAggregatorsWorker)->stop(); - } - catch (const std::bad_weak_ptr&) {} - } - } - /// - /// Main class - /// - ANSALPR_OV::ANSALPR_OV() {}; - ANSALPR_OV::~ANSALPR_OV() { - if (_detector == nullptr) { - delete _detector; - _detector = nullptr; - } - if (_vehicleAttributesClassifier == nullptr) { - delete _vehicleAttributesClassifier; - _vehicleAttributesClassifier = nullptr; - } - if (_lpr == nullptr) { - delete _lpr; - _lpr = nullptr; - } - }; - bool ANSALPR_OV::Destroy() { - if (_detector == nullptr) { - delete _detector; - _detector = nullptr; - } - if (_vehicleAttributesClassifier == nullptr) { - delete _vehicleAttributesClassifier; - _vehicleAttributesClassifier = nullptr; - } - if (_lpr == nullptr) { - delete _lpr; - _lpr = nullptr; - } - return true; - }; - bool ANSALPR_OV::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) { - try { - _licenseKey = licenseKey; - _licenseValid = false; - CheckLicense(); - if (!_licenseValid) { - this->_logger->LogError("ANSALPR_OV::Initialize.", "License is not valid.", __FILE__, __LINE__); - return false; - } - // Extract model folder - // 0. Check if the modelZipFilePath exist? - if (!FileExist(modelZipFilePath)) { - this->_logger->LogFatal("ANSALPR_OV::Initialize", "Model zip file is not exist", __FILE__, __LINE__); - } - // 1. Unzip model zip file to a special location with folder name as model file (and version) - std::string outputFolder; - std::vector passwordArray; - if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword); - passwordArray.push_back("AnsDemoModels20@!"); - passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag=="); - passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg=="); - std::string modelName = GetFileNameWithoutExtension(modelZipFilePath); - - size_t vectorSize = passwordArray.size(); - for (size_t i = 0; i < vectorSize; i++) { - if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false)) - break; // Break the loop when the condition is met. - } - // 2. Check if the outputFolder exist - if (!FolderExist(_modelFolder)) { - this->_logger->LogError("ANSALPR_OV::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__); - return false; // That means the model file is not exist or the password is not correct - } - - _vehicleLPModel = CreateFilePath(_modelFolder, "vehiclelp.xml"); - _vehicleAtModel = CreateFilePath(_modelFolder, "vehicle.xml"); - _lprModel = CreateFilePath(_modelFolder, "lpr.xml"); - - ov::Core core; - int FLAGS_nthreads = 0; - - std::set devices; - for (const std::string& netDevices : { "GPU", "GPU", "GPU" }) { - if (netDevices.empty()) { - continue; - } - for (const std::string& device : parseDevices(netDevices)) { - devices.insert(device); - } - } - - std::map device_nstreams = parseValuePerDevice(devices, ""); - - for (const std::string& device : devices) { - if ("CPU" == device) { - if (FLAGS_nthreads != 0) { - core.set_property("CPU", ov::inference_num_threads(FLAGS_nthreads)); - } - //core.set_property("CPU", ov::affinity(ov::Affinity::NONE)); - core.set_property("CPU", ov::streams::num((device_nstreams.count("CPU") > 0 ? ov::streams::Num(device_nstreams["CPU"]) : ov::streams::AUTO))); - - device_nstreams["CPU"] = core.get_property("CPU", ov::streams::num); - } - if ("GPU" == device) { - core.set_property("GPU", ov::streams::num(device_nstreams.count("GPU") > 0 ? ov::streams::Num(device_nstreams["GPU"]) : ov::streams::AUTO)); - - device_nstreams["GPU"] = core.get_property("GPU", ov::streams::num); - if (devices.end() != devices.find("CPU")) { - core.set_property("GPU", ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel(1))); - } - } - } - double FLAGS_t = 0.5; - if(FileExist(_vehicleLPModel))_detector = new Detector(core, "GPU", _vehicleLPModel, { static_cast(FLAGS_t), static_cast(FLAGS_t) }, false); - else { - this->_logger->LogFatal("ANSALPR_OV::Initialize", _vehicleLPModel, __FILE__, __LINE__); - - } - if(FileExist(_vehicleAtModel))_vehicleAttributesClassifier = new VehicleAttributesClassifier(core, "GPU", _vehicleAtModel, false); - else { - this->_logger->LogFatal("ANSALPR_OV::Initialize", _vehicleAtModel, __FILE__, __LINE__); - - } - if(FileExist(_lprModel)) _lpr = new Lpr(core, "CPU", _lprModel, false); - else { - this->_logger->LogFatal("ANSALPR_OV::Initialize", _lprModel, __FILE__, __LINE__); - - } - - if (FileExist(_vehicleLPModel) && - FileExist(_vehicleAtModel) && - FileExist(_lprModel))return true; - else return false; - } - catch (std::exception& e) { - this->_logger->LogFatal("ANSALPR_OV::Initialize", e.what(), __FILE__, __LINE__); - return false; - } - }; - bool ANSALPR_OV::Inference(const cv::Mat& input, std::string& lprResult) { - cv::Mat frame = input.clone(); - std::shared_ptr inputSource = std::make_shared(frame, true); - std::vector> _inputChannels; - _inputChannels.push_back(InputChannel::create(inputSource)); - unsigned nireq = 1; - bool isVideo = false; - std::size_t nclassifiersireq{ 0 }; - std::size_t nrecognizersireq{ 0 }; - nclassifiersireq = nireq * 3; - nrecognizersireq = nireq * 3; - - Context context = { _inputChannels, - *_detector, - *_vehicleAttributesClassifier, - *_lpr, - 2, - nireq, - isVideo, - nclassifiersireq, - nrecognizersireq }; - - std::shared_ptr worker = std::make_shared(2); - context.readersContext.readersWorker = worker; - context.inferTasksContext.inferTasksWorker = worker; - context.detectionsProcessorsContext.detectionsProcessorsWorker = worker; - context.resAggregatorsWorker = worker; - for (unsigned sourceID = 0; sourceID < _inputChannels.size(); sourceID++) { - VideoFrame::Ptr sharedVideoFrame = std::make_shared(context, sourceID, 0); - worker->push(std::make_shared(sharedVideoFrame)); - } - - // Running - worker->runThreads(); - worker->threadFunc(); - worker->join(); - - std::list boxesAndDescrs = context.boxesAndDescrs; - std::vector output; - output.clear(); - for(const BboxAndDescr boxesAndDescr: boxesAndDescrs) - { - if (boxesAndDescr.objectType == ANSCENTER::BboxAndDescr::ObjectType::PLATE) { - ALPRObject result; - result.classId = 0; - result.className = boxesAndDescr.descr; - result.confidence = 1.0; - result.box = boxesAndDescr.rect; - output.push_back(result); - } - } - lprResult = VectorDetectionToJsonString(output); - return true; - }; - bool ANSALPR_OV::Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult) { - return true; - } - std::string ANSALPR_OV::VectorDetectionToJsonString(const std::vector& dets) { - boost::property_tree::ptree root; - boost::property_tree::ptree detectedObjects; - for (int i = 0; i < dets.size(); i++) { - boost::property_tree::ptree detectedNode; - detectedNode.put("class_id", dets[i].classId); - detectedNode.put("class_name", dets[i].className); - detectedNode.put("prob", dets[i].confidence); - detectedNode.put("x", dets[i].box.x); - detectedNode.put("y", dets[i].box.y); - detectedNode.put("width", dets[i].box.width); - detectedNode.put("height", dets[i].box.height); - detectedNode.put("mask", "");//Todo: convert masks to mask with comma seperated dets[i].mask); - detectedNode.put("extra_info", ""); - - // we might add masks into this using comma seperated string - detectedObjects.push_back(std::make_pair("", detectedNode)); - } - root.add_child("results", detectedObjects); - std::ostringstream stream; - boost::property_tree::write_json(stream, root, false); - std::string trackingResult = stream.str(); - return trackingResult; - } -} \ No newline at end of file diff --git a/modules/ANSLPR/ANSLPR.h b/modules/ANSLPR/ANSLPR.h index c647702..c14f4fe 100644 --- a/modules/ANSLPR/ANSLPR.h +++ b/modules/ANSLPR/ANSLPR.h @@ -119,6 +119,9 @@ namespace ANSCENTER void SetALPRCheckerEnabled(bool enable) { _enableALPRChecker = enable; } bool IsALPRCheckerEnabled() const { return _enableALPRChecker; } + virtual void SetCountry(Country country) { _country = country; } + Country GetCountry() const { return _country; } + [[nodiscard]] virtual bool Destroy() = 0; [[nodiscard]] static std::vector GetBoundingBoxes(const std::string& strBBoxes); [[nodiscard]] static std::string PolygonToString(const std::vector& polygon); @@ -172,6 +175,9 @@ extern "C" ANSLPR_API int ANSALPR_GetFormats(ANSCENTER::ANSALPR** Handle, LS // ALPRChecker: 1 = enabled (full-frame auto-detected), 0 = disabled (raw OCR) extern "C" ANSLPR_API int ANSALPR_SetALPRCheckerEnabled(ANSCENTER::ANSALPR** Handle, int enable); +// Country: 0=VIETNAM, 1=CHINA, 2=AUSTRALIA, 3=USA, 4=INDONESIA, 5=JAPAN +extern "C" ANSLPR_API int ANSALPR_SetCountry(ANSCENTER::ANSALPR** Handle, int country); + // Unicode conversion utilities for LabVIEW wrapper classes extern "C" ANSLPR_API int ANSLPR_ConvertUTF8ToUTF16LE(const char* utf8Str, LStrHandle result, int includeBOM = 1); extern "C" ANSLPR_API int ANSLPR_ConvertUTF16LEToUTF8(const unsigned char* utf16leBytes, int byteLen, LStrHandle result); diff --git a/modules/ANSLPR/ANSLPR_OCR.cpp b/modules/ANSLPR/ANSLPR_OCR.cpp new file mode 100644 index 0000000..a61988b --- /dev/null +++ b/modules/ANSLPR/ANSLPR_OCR.cpp @@ -0,0 +1,665 @@ +#include "ANSLPR_OCR.h" +#include "ANSONNXYOLO.h" +#include "ANSOnnxOCR.h" +#include "ANSOCRBase.h" + +#include +#include +#include + +// --------------------------------------------------------------------------- +// SEH wrapper for loading ONNX models — identical to the one in ANSLPR_OD.cpp +// --------------------------------------------------------------------------- +static void WriteEventLog(const char* message, WORD eventType = EVENTLOG_INFORMATION_TYPE) { + static HANDLE hEventLog = RegisterEventSourceA(NULL, "ANSLogger"); + if (hEventLog) { + const char* msgs[1] = { message }; + ReportEventA(hEventLog, eventType, 0, 0, NULL, 1, 0, msgs, NULL); + } + OutputDebugStringA(message); + OutputDebugStringA("\n"); +} + +struct LoadOnnxParams_OCR { + const std::string* licenseKey; + ANSCENTER::ModelConfig* config; + const std::string* modelFolder; + const char* modelName; + const char* classFile; + std::string* labels; + std::unique_ptr* detector; + bool enableTracker; + bool disableStabilization; +}; + +static bool LoadOnnxModel_OCR_Impl(const LoadOnnxParams_OCR& p) { + try { + auto onnxyolo = std::make_unique(); + bool ok = onnxyolo->LoadModelFromFolder( + *p.licenseKey, *p.config, p.modelName, p.classFile, + *p.modelFolder, *p.labels); + if (!ok) { + return false; + } + if (p.enableTracker) { + onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, true); + } else { + onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, false); + } + if (p.disableStabilization) { + onnxyolo->SetStabilization(false); + } + *p.detector = std::move(onnxyolo); + return true; + } + catch (...) { + p.detector->reset(); + return false; + } +} + +static bool LoadOnnxModel_OCR_SEH(const LoadOnnxParams_OCR& p, DWORD* outCode) { + *outCode = 0; + __try { + return LoadOnnxModel_OCR_Impl(p); + } + __except (EXCEPTION_EXECUTE_HANDLER) { + *outCode = GetExceptionCode(); + return false; + } +} + +namespace ANSCENTER +{ + ANSALPR_OCR::ANSALPR_OCR() { + engineType = EngineType::CPU; + } + + ANSALPR_OCR::~ANSALPR_OCR() { + try { + Destroy(); + } + catch (...) {} + } + + bool ANSALPR_OCR::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, + const std::string& modelZipPassword, double detectorThreshold, double ocrThreshold, double colourThreshold) { + std::lock_guard lock(_mutex); + try { + _licenseKey = licenseKey; + _licenseValid = false; + _detectorThreshold = detectorThreshold; + _ocrThreshold = ocrThreshold; + _colorThreshold = colourThreshold; + _country = Country::JAPAN; // Default to JAPAN for OCR-based ALPR + CheckLicense(); + if (!_licenseValid) { + this->_logger.LogError("ANSALPR_OCR::Initialize", "License is not valid.", __FILE__, __LINE__); + return false; + } + + // Extract model folder + if (!FileExist(modelZipFilePath)) { + this->_logger.LogFatal("ANSALPR_OCR::Initialize", "Model zip file does not exist: " + modelZipFilePath, __FILE__, __LINE__); + return false; + } + this->_logger.LogInfo("ANSALPR_OCR::Initialize", "Model zip file found: " + modelZipFilePath, __FILE__, __LINE__); + + // Unzip model zip file + std::vector passwordArray; + if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword); + passwordArray.push_back("AnsDemoModels20@!"); + passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag=="); + passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg=="); + std::string modelName = GetFileNameWithoutExtension(modelZipFilePath); + + for (size_t i = 0; i < passwordArray.size(); i++) { + if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false)) + break; + } + + if (!FolderExist(_modelFolder)) { + this->_logger.LogError("ANSALPR_OCR::Initialize", "Output model folder does not exist: " + _modelFolder, __FILE__, __LINE__); + return false; + } + + // Check country from country.txt + std::string countryFile = CreateFilePath(_modelFolder, "country.txt"); + if (FileExist(countryFile)) { + std::ifstream infile(countryFile); + std::string countryStr; + std::getline(infile, countryStr); + infile.close(); + if (countryStr == "0") _country = Country::VIETNAM; + else if (countryStr == "1") _country = Country::CHINA; + else if (countryStr == "2") _country = Country::AUSTRALIA; + else if (countryStr == "3") _country = Country::USA; + else if (countryStr == "4") _country = Country::INDONESIA; + else if (countryStr == "5") _country = Country::JAPAN; + else _country = Country::JAPAN; // Default for OCR mode + } + + // Store the original model zip path — the OCR models (ansocrdec.onnx, + // ansocrcls.onnx, ansocrrec.onnx, dict_ch.txt) are bundled inside the + // same ALPR model zip, so we reuse it for ANSONNXOCR initialization. + _modelZipFilePath = modelZipFilePath; + + // Initialize ALPRChecker + alprChecker.Init(MAX_ALPR_FRAME); + + _lpColourModelConfig.detectionScoreThreshold = _colorThreshold; + _lpdmodelConfig.detectionScoreThreshold = _detectorThreshold; + + return true; + } + catch (std::exception& e) { + this->_logger.LogFatal("ANSALPR_OCR::Initialize", e.what(), __FILE__, __LINE__); + return false; + } + } + + bool ANSALPR_OCR::LoadEngine() { + std::lock_guard lock(_mutex); + try { + WriteEventLog("ANSALPR_OCR::LoadEngine: Step 1 - Starting engine load"); + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 1: Starting engine load", __FILE__, __LINE__); + + // Detect hardware + _lpdmodelConfig.detectionScoreThreshold = _detectorThreshold; + _lpColourModelConfig.detectionScoreThreshold = _colorThreshold; + + if (_lpdmodelConfig.detectionScoreThreshold < 0.25) _lpdmodelConfig.detectionScoreThreshold = 0.25; + if (_lpdmodelConfig.detectionScoreThreshold > 0.95) _lpdmodelConfig.detectionScoreThreshold = 0.95; + + engineType = ANSLicenseHelper::CheckHardwareInformation(); + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Detected engine type: " + std::to_string(static_cast(engineType)), __FILE__, __LINE__); + + float confThreshold = 0.5f; + float MNSThreshold = 0.5f; + _lpdmodelConfig.modelConfThreshold = confThreshold; + _lpdmodelConfig.modelMNSThreshold = MNSThreshold; + _lpColourModelConfig.modelConfThreshold = confThreshold; + _lpColourModelConfig.modelMNSThreshold = MNSThreshold; + + std::string lprModel = CreateFilePath(_modelFolder, "lpd.onnx"); + std::string colorModel = CreateFilePath(_modelFolder, "lpc.onnx"); + + bool valid = false; + + // ── Step 2: Load LP detector with ONNX Runtime ─────────────── + if (FileExist(lprModel)) { + WriteEventLog("ANSALPR_OCR::LoadEngine: Step 2 - Loading LP detector with ONNX Runtime"); + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 2: Loading LP detector with ONNX Runtime", __FILE__, __LINE__); + _lpdmodelConfig.detectionType = DetectionType::DETECTION; + _lpdmodelConfig.modelType = ModelType::ONNXYOLO; + std::string _lprClasses; + { + LoadOnnxParams_OCR p{}; + p.licenseKey = &_licenseKey; + p.config = &_lpdmodelConfig; + p.modelFolder = &_modelFolder; + p.modelName = "lpd"; + p.classFile = "lpd.names"; + p.labels = &_lprClasses; + p.detector = &_lpDetector; + p.enableTracker = true; + p.disableStabilization = true; + + DWORD sehCode = 0; + bool lpSuccess = LoadOnnxModel_OCR_SEH(p, &sehCode); + if (sehCode != 0) { + char buf[256]; + snprintf(buf, sizeof(buf), + "ANSALPR_OCR::LoadEngine: Step 2 LPD SEH exception 0x%08X — LP detector disabled", sehCode); + WriteEventLog(buf, EVENTLOG_ERROR_TYPE); + this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", + "Step 2: LP detector crashed (SEH). LP detector disabled.", __FILE__, __LINE__); + if (_lpDetector) _lpDetector.reset(); + } + else if (!lpSuccess) { + this->_logger.LogError("ANSALPR_OCR::LoadEngine", + "Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__); + if (_lpDetector) _lpDetector.reset(); + } + } + } + + if (!_lpDetector) { + this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", "LP detector failed to load. Cannot proceed.", __FILE__, __LINE__); + _isInitialized = false; + return false; + } + + // ── Step 3: Load OCR engine (ANSONNXOCR) ───────────────────── + // The OCR models (ansocrdec.onnx, ansocrcls.onnx, ansocrrec.onnx, + // dict_ch.txt) are bundled inside the same ALPR model zip, so we + // pass the original ALPR zip path to ANSONNXOCR::Initialize. + // ANSOCRBase::Initialize will extract it (no-op if already done) + // and discover the OCR model files in the extracted folder. + WriteEventLog("ANSALPR_OCR::LoadEngine: Step 3 - Loading OCR engine (ANSONNXOCR)"); + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 3: Loading OCR engine (ANSONNXOCR)", __FILE__, __LINE__); + + // Verify OCR model files exist in the already-extracted folder + std::string ocrDetModel = CreateFilePath(_modelFolder, "ansocrdec.onnx"); + std::string ocrRecModel = CreateFilePath(_modelFolder, "ansocrrec.onnx"); + if (!FileExist(ocrDetModel) || !FileExist(ocrRecModel)) { + this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", + "OCR model files not found in model folder: " + _modelFolder + + " (expected ansocrdec.onnx, ansocrrec.onnx)", __FILE__, __LINE__); + _isInitialized = false; + return false; + } + + _ocrEngine = std::make_unique(); + + // Determine OCR language based on country + OCRLanguage ocrLang = OCRLanguage::ENGLISH; + switch (_country) { + case Country::JAPAN: ocrLang = OCRLanguage::JAPANESE; break; + case Country::CHINA: ocrLang = OCRLanguage::CHINESE; break; + case Country::VIETNAM: ocrLang = OCRLanguage::ENGLISH; break; + case Country::AUSTRALIA: ocrLang = OCRLanguage::ENGLISH; break; + case Country::USA: ocrLang = OCRLanguage::ENGLISH; break; + case Country::INDONESIA: ocrLang = OCRLanguage::ENGLISH; break; + default: ocrLang = OCRLanguage::ENGLISH; break; + } + + OCRModelConfig ocrModelConfig; + ocrModelConfig.ocrLanguage = ocrLang; + ocrModelConfig.useDetector = true; + ocrModelConfig.useRecognizer = true; + ocrModelConfig.useCLS = true; + ocrModelConfig.useLayout = false; + ocrModelConfig.useTable = false; + ocrModelConfig.useTensorRT = false; + ocrModelConfig.enableMKLDNN = false; + ocrModelConfig.useDilation = true; + ocrModelConfig.useAngleCLS = false; + ocrModelConfig.gpuId = 0; + ocrModelConfig.detectionDBThreshold = 0.5; + ocrModelConfig.detectionBoxThreshold = 0.3; + ocrModelConfig.detectionDBUnclipRatio = 1.2; + ocrModelConfig.clsThreshold = 0.9; + ocrModelConfig.limitSideLen = 2560; + + // Pass the original ALPR model zip path — ANSOCRBase::Initialize + // will extract it to the same folder (already done, so extraction + // is a no-op) and set up ansocrdec.onnx / ansocrcls.onnx / + // ansocrrec.onnx / dict_ch.txt paths automatically. + bool ocrSuccess = _ocrEngine->Initialize(_licenseKey, ocrModelConfig, _modelZipFilePath, "", 0); + if (!ocrSuccess) { + this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", "Failed to initialize OCR engine (ANSONNXOCR).", __FILE__, __LINE__); + _ocrEngine.reset(); + _isInitialized = false; + return false; + } + + // Set ALPR mode and country on the OCR engine + _ocrEngine->SetOCRMode(OCRMode::OCR_ALPR); + _ocrEngine->SetCountry(_country); + + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 3: OCR engine loaded successfully.", __FILE__, __LINE__); + + // ── Step 4: Load colour classifier (optional) ──────────────── + if (FileExist(colorModel) && (_lpColourModelConfig.detectionScoreThreshold > 0)) { + WriteEventLog("ANSALPR_OCR::LoadEngine: Step 4 - Loading colour classifier with ONNX Runtime"); + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 4: Loading colour classifier with ONNX Runtime", __FILE__, __LINE__); + _lpColourModelConfig.detectionType = DetectionType::CLASSIFICATION; + _lpColourModelConfig.modelType = ModelType::ONNXYOLO; + { + LoadOnnxParams_OCR p{}; + p.licenseKey = &_licenseKey; + p.config = &_lpColourModelConfig; + p.modelFolder = &_modelFolder; + p.modelName = "lpc"; + p.classFile = "lpc.names"; + p.labels = &_lpColourLabels; + p.detector = &_lpColourDetector; + p.enableTracker = false; + p.disableStabilization = false; + + DWORD sehCode = 0; + bool colourSuccess = LoadOnnxModel_OCR_SEH(p, &sehCode); + if (sehCode != 0) { + char buf[256]; + snprintf(buf, sizeof(buf), + "ANSALPR_OCR::LoadEngine: Step 4 LPC SEH exception 0x%08X — colour detection disabled", sehCode); + WriteEventLog(buf, EVENTLOG_ERROR_TYPE); + this->_logger.LogError("ANSALPR_OCR::LoadEngine", + "Step 4: Colour classifier crashed. Colour detection disabled.", __FILE__, __LINE__); + if (_lpColourDetector) _lpColourDetector.reset(); + } + else if (!colourSuccess) { + this->_logger.LogError("ANSALPR_OCR::LoadEngine", + "Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__); + if (_lpColourDetector) _lpColourDetector.reset(); + } + } + } + + valid = true; + _isInitialized = valid; + WriteEventLog(("ANSALPR_OCR::LoadEngine: Step 5 - Engine load complete. Valid = " + std::to_string(valid)).c_str()); + this->_logger.LogInfo("ANSALPR_OCR::LoadEngine", "Step 5: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__); + return valid; + } + catch (std::exception& e) { + WriteEventLog(("ANSALPR_OCR::LoadEngine: C++ exception: " + std::string(e.what())).c_str(), EVENTLOG_ERROR_TYPE); + this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", std::string("C++ exception: ") + e.what(), __FILE__, __LINE__); + _isInitialized = false; + return false; + } + catch (...) { + WriteEventLog("ANSALPR_OCR::LoadEngine: Unknown exception", EVENTLOG_ERROR_TYPE); + this->_logger.LogFatal("ANSALPR_OCR::LoadEngine", "Unknown exception", __FILE__, __LINE__); + _isInitialized = false; + return false; + } + } + + // ── Colour detection (same pattern as ANSALPR_OD) ──────────────────── + std::string ANSALPR_OCR::DetectLPColourDetector(const cv::Mat& lprROI, const std::string& cameraId) { + if (_lpColourModelConfig.detectionScoreThreshold <= 0.0f) return {}; + if (!_lpColourDetector) return {}; + if (lprROI.empty()) return {}; + + try { + std::vector colourOutputs = _lpColourDetector->RunInference(lprROI, cameraId); + if (colourOutputs.empty()) return {}; + + const auto& bestDetection = *std::max_element( + colourOutputs.begin(), colourOutputs.end(), + [](const Object& a, const Object& b) { return a.confidence < b.confidence; } + ); + return bestDetection.className; + } + catch (const std::exception& e) { + this->_logger.LogFatal("ANSALPR_OCR::DetectLPColourDetector", e.what(), __FILE__, __LINE__); + return {}; + } + } + + std::string ANSALPR_OCR::DetectLPColourCached(const cv::Mat& lprROI, const std::string& cameraId, const std::string& plateText) { + if (plateText.empty()) { + return DetectLPColourDetector(lprROI, cameraId); + } + + // Check cache first + { + std::lock_guard cacheLock(_colourCacheMutex); + auto it = _colourCache.find(plateText); + if (it != _colourCache.end()) { + it->second.hitCount++; + return it->second.colour; + } + } + + // Cache miss — run classifier + std::string colour = DetectLPColourDetector(lprROI, cameraId); + + if (!colour.empty()) { + std::lock_guard cacheLock(_colourCacheMutex); + if (_colourCache.size() >= COLOUR_CACHE_MAX_SIZE) { + _colourCache.clear(); + } + _colourCache[plateText] = { colour, 0 }; + } + + return colour; + } + + // ── OCR on a single plate ROI ──────────────────────────────────────── + // Returns the plate text via the out-parameter and populates alprExtraInfo + // with the structured ALPR JSON (zone parts) when ALPR mode is active. + std::string ANSALPR_OCR::RunOCROnPlate(const cv::Mat& plateROI, const std::string& cameraId) { + if (!_ocrEngine || plateROI.empty()) return ""; + if (plateROI.cols < 10 || plateROI.rows < 10) return ""; + + try { + // Run the full ANSONNXOCR pipeline on the cropped plate image + std::vector ocrResults = _ocrEngine->RunInference(plateROI, cameraId); + + if (ocrResults.empty()) return ""; + + // If ALPR mode is active and we have plate formats, use the + // structured ALPR post-processing to get correct zone ordering + // (e.g. "品川 302 ま 93-15" instead of "品川30293-15ま") + const auto& alprFormats = _ocrEngine->GetALPRFormats(); + if (_ocrEngine->GetOCRMode() == OCRMode::OCR_ALPR && !alprFormats.empty()) { + auto alprResults = ANSOCRUtility::ALPRPostProcessing( + ocrResults, alprFormats, + plateROI.cols, plateROI.rows, + _ocrEngine.get(), plateROI); + + if (!alprResults.empty()) { + return alprResults[0].fullPlateText; + } + } + + // Fallback: simple concatenation sorted by Y then X + std::sort(ocrResults.begin(), ocrResults.end(), + [](const OCRObject& a, const OCRObject& b) { + int rowThreshold = std::min(a.box.height, b.box.height) / 2; + if (std::abs(a.box.y - b.box.y) > rowThreshold) { + return a.box.y < b.box.y; + } + return a.box.x < b.box.x; + } + ); + + std::string fullText; + for (const auto& obj : ocrResults) { + if (!obj.className.empty()) { + fullText += obj.className; + } + } + + return fullText; + } + catch (const std::exception& e) { + this->_logger.LogError("ANSALPR_OCR::RunOCROnPlate", e.what(), __FILE__, __LINE__); + return ""; + } + } + + // ── Main inference pipeline ────────────────────────────────────────── + std::vector ANSALPR_OCR::RunInference(const cv::Mat& input, const std::string& cameraId) { + if (!_licenseValid) { + this->_logger.LogError("ANSALPR_OCR::RunInference", "Invalid license", __FILE__, __LINE__); + return {}; + } + if (!_isInitialized) { + this->_logger.LogError("ANSALPR_OCR::RunInference", "Model is not initialized", __FILE__, __LINE__); + return {}; + } + if (input.empty() || input.cols < 5 || input.rows < 5) { + this->_logger.LogError("ANSALPR_OCR::RunInference", "Input image is empty or too small", __FILE__, __LINE__); + return {}; + } + if (!_lpDetector) { + this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_lpDetector is null", __FILE__, __LINE__); + return {}; + } + if (!_ocrEngine) { + this->_logger.LogFatal("ANSALPR_OCR::RunInference", "_ocrEngine is null", __FILE__, __LINE__); + return {}; + } + + try { + // Convert grayscale to BGR if necessary + cv::Mat localFrame; + if (input.channels() == 1) { + cv::cvtColor(input, localFrame, cv::COLOR_GRAY2BGR); + } + const cv::Mat& frame = (input.channels() == 1) ? localFrame : input; + + const int frameWidth = frame.cols; + const int frameHeight = frame.rows; + + // Step 1: Detect license plates + std::vector lprOutput = _lpDetector->RunInference(frame, cameraId); + + if (lprOutput.empty()) { + return {}; + } + + std::vector output; + output.reserve(lprOutput.size()); + + for (auto& lprObject : lprOutput) { + const cv::Rect& box = lprObject.box; + + // Calculate safe cropped region + const int x1 = std::max(0, box.x); + const int y1 = std::max(0, box.y); + const int width = std::min(frameWidth - x1, box.width); + const int height = std::min(frameHeight - y1, box.height); + + if (width <= 0 || height <= 0) continue; + + cv::Rect lprPos(x1, y1, width, height); + cv::Mat plateROI = frame(lprPos); + + // Step 2: Run OCR on the detected plate + std::string ocrText = RunOCROnPlate(plateROI, cameraId); + + if (ocrText.empty()) continue; + + lprObject.cameraId = cameraId; + + // Use ALPRChecker for text stabilization if enabled + if (_enableALPRChecker) { + lprObject.className = alprChecker.checkPlateByTrackId(cameraId, ocrText, lprObject.trackId); + } else { + lprObject.className = ocrText; + } + + if (lprObject.className.empty()) continue; + + // Step 3: Colour detection (optional) + std::string colour = DetectLPColourCached(plateROI, cameraId, lprObject.className); + if (!colour.empty()) { + lprObject.extraInfo = "color:" + colour; + } + + output.push_back(std::move(lprObject)); + } + + return output; + } + catch (const cv::Exception& e) { + this->_logger.LogFatal("ANSALPR_OCR::RunInference", std::string("OpenCV Exception: ") + e.what(), __FILE__, __LINE__); + } + catch (const std::exception& e) { + this->_logger.LogFatal("ANSALPR_OCR::RunInference", e.what(), __FILE__, __LINE__); + } + catch (...) { + this->_logger.LogFatal("ANSALPR_OCR::RunInference", "Unknown exception occurred", __FILE__, __LINE__); + } + + return {}; + } + + // ── Inference wrappers ─────────────────────────────────────────────── + bool ANSALPR_OCR::Inference(const cv::Mat& input, std::string& lprResult) { + if (input.empty()) return false; + if (input.cols < 5 || input.rows < 5) return false; + return Inference(input, lprResult, "CustomCam"); + } + + bool ANSALPR_OCR::Inference(const cv::Mat& input, std::string& lprResult, const std::string& cameraId) { + if (input.empty()) return false; + if (input.cols < 5 || input.rows < 5) return false; + + try { + std::vector results = RunInference(input, cameraId); + lprResult = VectorDetectionToJsonString(results); + return !results.empty(); + } + catch (...) { + return false; + } + } + + bool ANSALPR_OCR::Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult) { + return Inference(input, Bbox, lprResult, "CustomCam"); + } + + bool ANSALPR_OCR::Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult, const std::string& cameraId) { + if (input.empty()) return false; + if (input.cols < 5 || input.rows < 5) return false; + + try { + if (Bbox.empty()) { + return Inference(input, lprResult, cameraId); + } + + // For cropped images, run OCR on each bounding box + std::vector allResults; + cv::Mat frame; + if (input.channels() == 1) { + cv::cvtColor(input, frame, cv::COLOR_GRAY2BGR); + } else { + frame = input; + } + + for (const auto& bbox : Bbox) { + int x1 = std::max(0, bbox.x); + int y1 = std::max(0, bbox.y); + int w = std::min(frame.cols - x1, bbox.width); + int h = std::min(frame.rows - y1, bbox.height); + + if (w < 5 || h < 5) continue; + + cv::Rect safeRect(x1, y1, w, h); + cv::Mat cropped = frame(safeRect); + + std::vector results = RunInference(cropped, cameraId); + + // Adjust bounding boxes back to full image coordinates + for (auto& obj : results) { + obj.box.x += x1; + obj.box.y += y1; + allResults.push_back(std::move(obj)); + } + } + + lprResult = VectorDetectionToJsonString(allResults); + return !allResults.empty(); + } + catch (...) { + return false; + } + } + + void ANSALPR_OCR::SetCountry(Country country) { + _country = country; + if (_ocrEngine) { + _ocrEngine->SetCountry(country); + } + } + + bool ANSALPR_OCR::Destroy() { + try { + if (_lpDetector) { + _lpDetector->Destroy(); + _lpDetector.reset(); + } + if (_lpColourDetector) { + _lpColourDetector->Destroy(); + _lpColourDetector.reset(); + } + if (_ocrEngine) { + _ocrEngine->Destroy(); + _ocrEngine.reset(); + } + _isInitialized = false; + return true; + } + catch (std::exception& e) { + this->_logger.LogFatal("ANSALPR_OCR::Destroy", e.what(), __FILE__, __LINE__); + return false; + } + } + +} // namespace ANSCENTER diff --git a/modules/ANSLPR/ANSLPR_OCR.h b/modules/ANSLPR/ANSLPR_OCR.h new file mode 100644 index 0000000..c9dc8f3 --- /dev/null +++ b/modules/ANSLPR/ANSLPR_OCR.h @@ -0,0 +1,91 @@ +#ifndef ANSLPROCR_H +#define ANSLPROCR_H +#pragma once +#include "ANSLPR.h" +#include +#include +#include +#include +#include +#include + +// Forward-declare ANSONNXOCR to avoid pulling in the full ANSOCR header chain +namespace ANSCENTER { class ANSONNXOCR; struct OCRModelConfig; } + +namespace ANSCENTER +{ + /// ANSALPR_OCR — License plate recognition using ONNX YOLO for LP detection + /// and ANSONNXOCR (PaddleOCR v5) for text recognition. + /// + /// Pipeline: + /// 1. Detect license plates using _lpDetector (ANSONNXYOLO) + /// 2. For each detected plate, run OCR using _ocrEngine (ANSONNXOCR) + /// 3. Optionally classify plate colour using _lpColourDetector (ANSONNXYOLO) + /// + /// Supports multiple countries via the Country enum and ALPR post-processing + /// from ANSOCR's ANSOCRBase infrastructure. + class ANSLPR_API ANSALPR_OCR : public ANSALPR { + private: + ANSCENTER::EngineType engineType; + + // --- Detectors --- + std::unique_ptr _lpDetector = nullptr; // License plate detector + std::unique_ptr _lpColourDetector = nullptr; // License plate colour classifier + std::unique_ptr _ocrEngine = nullptr; // OCR text recognizer + + // --- Model configs --- + ANSCENTER::ModelConfig _lpdmodelConfig; + ANSCENTER::ModelConfig _lpColourModelConfig; + + std::string _lpdLabels; + std::string _lpColourLabels; + cv::Mat _frameBuffer; // Reusable buffer for color conversion + + std::vector _lprModelClass; + + ALPRChecker alprChecker; + + // --- Original model zip path (reused for ANSONNXOCR initialization) --- + std::string _modelZipFilePath; + + // --- Colour detection helpers --- + [[nodiscard]] std::string DetectLPColourDetector(const cv::Mat& lprROI, const std::string& cameraId); + [[nodiscard]] std::string DetectLPColourCached(const cv::Mat& lprROI, const std::string& cameraId, const std::string& plateText); + + // LPC colour cache + struct ColourCacheEntry { + std::string colour; + int hitCount = 0; + }; + std::mutex _colourCacheMutex; + std::unordered_map _colourCache; + static constexpr size_t COLOUR_CACHE_MAX_SIZE = 200; + + // --- OCR helper --- + [[nodiscard]] std::string RunOCROnPlate(const cv::Mat& plateROI, const std::string& cameraId); + + public: + ANSALPR_OCR(); + ~ANSALPR_OCR(); + [[nodiscard]] bool Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword, double detectorThreshold, double ocrThreshold, double colourThreshold) override; + [[nodiscard]] bool LoadEngine() override; + [[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult) override; + [[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult, const std::string& cameraId) override; + [[nodiscard]] bool Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult) override; + [[nodiscard]] bool Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult, const std::string& cameraId) override; + [[nodiscard]] std::vector RunInference(const cv::Mat& input, const std::string& cameraId) override; + [[nodiscard]] bool Destroy() override; + + /// Propagate country to inner OCR engine so ALPR post-processing + /// uses the correct plate formats and character corrections. + void SetCountry(Country country) override; + + /// Propagate debug flag to all sub-detectors + void ActivateDebugger(bool debugFlag) override { + _debugFlag = debugFlag; + if (_lpDetector) _lpDetector->ActivateDebugger(debugFlag); + if (_lpColourDetector) _lpColourDetector->ActivateDebugger(debugFlag); + } + }; +} +#endif diff --git a/modules/ANSLPR/ANSLPR_OD.cpp b/modules/ANSLPR/ANSLPR_OD.cpp index 56e7685..42db6e2 100644 --- a/modules/ANSLPR/ANSLPR_OD.cpp +++ b/modules/ANSLPR/ANSLPR_OD.cpp @@ -272,6 +272,82 @@ static bool LoadLpcModel_SEH(const LoadLpcParams& p, DWORD* outCode) { return false; } } + +// --------------------------------------------------------------------------- +// Generic SEH wrapper for loading an ANSONNXYOLO model (used by the CPU / +// AMD / Intel fallback path where TensorRT is unavailable). +// +// Why SEH is required here +// ------------------------ +// DirectML / OpenVINO / CUDA ORT session creation can crash with an +// asynchronous hardware fault (STATUS_ACCESS_VIOLATION 0xC0000005) when +// the underlying provider driver is in a bad state. C++ `try/catch` does +// NOT catch SEH exceptions on MSVC unless the translator is explicitly +// installed. Without this SEH wrapper the AV propagates up through +// ANSALPR_OD::LoadEngine into LoadANSALPREngineHandle, which logs +// "SEH exception 0xC0000005 caught during engine load" and returns 0 — +// the user sees a generic error with no way to tell which detector +// (LPD / OCR / LPC) failed. +// +// Wrapping each detector creation lets us: +// 1. Isolate the failing detector without taking down the whole load. +// 2. Log a precise error message indicating which model crashed. +// 3. Let the caller zero out the unique_ptr so Destroy() won't run a +// half-initialised engine during cleanup. +// --------------------------------------------------------------------------- +struct LoadOnnxParams { + const std::string* licenseKey; + ANSCENTER::ModelConfig* config; + const std::string* modelFolder; + const char* modelName; + const char* classFile; + std::string* labels; + std::unique_ptr* detector; + bool enableTracker; + bool disableStabilization; +}; + +static bool LoadOnnxModel_Impl(const LoadOnnxParams& p) { + try { + auto onnxyolo = std::make_unique(); + bool ok = onnxyolo->LoadModelFromFolder( + *p.licenseKey, *p.config, p.modelName, p.classFile, + *p.modelFolder, *p.labels); + if (!ok) { + return false; + } + if (p.enableTracker) { + onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, true); + } else { + onnxyolo->SetTracker(ANSCENTER::TrackerType::BYTETRACK, false); + } + if (p.disableStabilization) { + onnxyolo->SetStabilization(false); + } + *p.detector = std::move(onnxyolo); // upcast ANSONNXYOLO -> ANSODBase + return true; + } + catch (...) { + p.detector->reset(); + return false; + } +} + +static bool LoadOnnxModel_SEH(const LoadOnnxParams& p, DWORD* outCode) { + // IMPORTANT: a function containing __try/__except must not run C++ + // destructors in the handler body — the CRT's SEH unwind can collide + // with C++ unwind and call std::terminate. We therefore defer any + // cleanup (unique_ptr::reset) to the caller, which runs outside the + // SEH context. This mirrors the LoadLpcModel_SEH pattern above. + *outCode = 0; + __try { + return LoadOnnxModel_Impl(p); + } + __except (EXCEPTION_EXECUTE_HANDLER) { + *outCode = GetExceptionCode(); + return false; + } +} //#define FNS_DEBUG namespace ANSCENTER { @@ -285,6 +361,12 @@ namespace ANSCENTER { ANSALPR_OD::ANSALPR_OD() { valid = false; + // Default to safest engine (CPU). LoadEngine() overrides this after + // CheckHardwareInformation() runs. We must not leave engineType + // uninitialised because vendor predicates (isNvidiaEngine() etc.) + // gate NV12/CUDA paths and could otherwise activate the CUDA runtime + // on AMD/Intel hardware. + engineType = ANSCENTER::EngineType::CPU; }; ANSALPR_OD::~ANSALPR_OD() { try { @@ -485,8 +567,16 @@ namespace ANSCENTER { WriteEventLog("ANSALPR_OD::LoadEngine: Step 2 - Checking hardware information"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 2: Checking hardware information", __FILE__, __LINE__); engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation();// - WriteEventLog(("ANSALPR_OD::LoadEngine: Step 2 complete - Engine type = " + std::to_string(static_cast(engineType))).c_str()); - this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 2 complete: Engine type = " + std::to_string(static_cast(engineType)), __FILE__, __LINE__); + const char* vendorTag = + isNvidiaEngine() ? "NVIDIA_GPU (TensorRT + NV12/CUDA fast path)" : + isAmdEngine() ? "AMD_GPU (DirectML via ONNX Runtime, NV12/CUDA DISABLED)" : + isIntelEngine() ? "OPENVINO_GPU (OpenVINO via ONNX Runtime, NV12/CUDA DISABLED)" : + "CPU (ONNX Runtime, NV12/CUDA DISABLED)"; + WriteEventLog(("ANSALPR_OD::LoadEngine: Step 2 complete - Engine type = " + + std::to_string(static_cast(engineType)) + " [" + vendorTag + "]").c_str()); + this->_logger.LogInfo("ANSALPR_OD::LoadEngine", + "Step 2 complete: Engine type = " + std::to_string(static_cast(engineType)) + + " [" + vendorTag + "]", __FILE__, __LINE__); valid = false; if (_lpDetector) _lpDetector.reset(); @@ -788,54 +878,132 @@ namespace ANSCENTER { } } } - // ONNX Runtime fallback path (CPU or when TensorRT fails) + // ONNX Runtime fallback path (CPU / AMD / Intel — or NVIDIA when + // TensorRT build failed). Each detector is loaded through a + // dedicated SEH wrapper (LoadOnnxModel_SEH) so that an + // AV / STATUS_ACCESS_VIOLATION raised deep inside the ONNX + // Runtime session creator (e.g. from a misbehaving DirectML + // / OpenVINO / CUDA provider driver) does not tear down the + // whole LoadEngine call. The wrapper logs the exact detector + // that failed and zeros out the corresponding unique_ptr. if (!valid) { if (FileExist(lprModel) && (FileExist(ocrModel))) { + bool lpSuccess = false, ocrSuccess = false; + + // ── Step 6: LPD ───────────────────────────────────── WriteEventLog("ANSALPR_OD::LoadEngine: Step 6 - Loading LP detector with ONNX Runtime"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 6: Loading LP detector with ONNX Runtime", __FILE__, __LINE__); _lpdmodelConfig.detectionType = DetectionType::DETECTION; - _lpdmodelConfig.modelType = ModelType::ONNXYOLO; + _lpdmodelConfig.modelType = ModelType::ONNXYOLO; std::string _lprClasses; - _lpDetector = std::make_unique();// Yolo - bool lpSuccess = _lpDetector->LoadModelFromFolder(_licenseKey, _lpdmodelConfig, "lpd", "lpd.names", _modelFolder, _lprClasses); - if (!lpSuccess) { - this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__); - _lpDetector.reset(); - } - else { - // Enable tracker on LP detector for stable bounding box tracking, - // but disable stabilization (no ghost plates — ALPRChecker handles text stabilization) - _lpDetector->SetTracker(TrackerType::BYTETRACK, true); - _lpDetector->SetStabilization(false); + { + LoadOnnxParams p{}; + p.licenseKey = &_licenseKey; + p.config = &_lpdmodelConfig; + p.modelFolder = &_modelFolder; + p.modelName = "lpd"; + p.classFile = "lpd.names"; + p.labels = &_lprClasses; + p.detector = &_lpDetector; + p.enableTracker = true; + p.disableStabilization = true; + + DWORD sehCode = 0; + lpSuccess = LoadOnnxModel_SEH(p, &sehCode); + if (sehCode != 0) { + char buf[256]; + snprintf(buf, sizeof(buf), + "ANSALPR_OD::LoadEngine: Step 6 LPD SEH exception 0x%08X — LP detector disabled", sehCode); + WriteEventLog(buf, EVENTLOG_ERROR_TYPE); + this->_logger.LogFatal("ANSALPR_OD::LoadEngine", + "Step 6: LP detector crashed (SEH 0x" + std::to_string(sehCode) + "). LP detector disabled.", + __FILE__, __LINE__); + lpSuccess = false; + // Drop any half-initialised state outside SEH context. + if (_lpDetector) _lpDetector.reset(); + } + else if (!lpSuccess) { + this->_logger.LogError("ANSALPR_OD::LoadEngine", + "Failed to load LP detector (ONNX Runtime).", __FILE__, __LINE__); + if (_lpDetector) _lpDetector.reset(); + } } + // ── Step 7: OCR ───────────────────────────────────── WriteEventLog("ANSALPR_OD::LoadEngine: Step 7 - Loading OCR detector with ONNX Runtime"); this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 7: Loading OCR detector with ONNX Runtime", __FILE__, __LINE__); _ocrModelConfig.detectionType = DetectionType::DETECTION; - _ocrModelConfig.modelType = ModelType::ONNXYOLO; - _ocrDetector = std::make_unique();// Yolo - bool ocrSuccess = _ocrDetector->LoadModelFromFolder(_licenseKey, _ocrModelConfig, "ocr", "ocr.names", _modelFolder, _ocrLabels); - if (!ocrSuccess) { - this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load OCR detector (ONNX Runtime).", __FILE__, __LINE__); - _ocrDetector.reset(); - } - else { - _ocrDetector->SetTracker(TrackerType::BYTETRACK, false); + _ocrModelConfig.modelType = ModelType::ONNXYOLO; + { + LoadOnnxParams p{}; + p.licenseKey = &_licenseKey; + p.config = &_ocrModelConfig; + p.modelFolder = &_modelFolder; + p.modelName = "ocr"; + p.classFile = "ocr.names"; + p.labels = &_ocrLabels; + p.detector = &_ocrDetector; + p.enableTracker = false; + p.disableStabilization = false; + + DWORD sehCode = 0; + ocrSuccess = LoadOnnxModel_SEH(p, &sehCode); + if (sehCode != 0) { + char buf[256]; + snprintf(buf, sizeof(buf), + "ANSALPR_OD::LoadEngine: Step 7 OCR SEH exception 0x%08X — OCR detector disabled", sehCode); + WriteEventLog(buf, EVENTLOG_ERROR_TYPE); + this->_logger.LogFatal("ANSALPR_OD::LoadEngine", + "Step 7: OCR detector crashed (SEH 0x" + std::to_string(sehCode) + "). OCR detector disabled.", + __FILE__, __LINE__); + ocrSuccess = false; + // Drop any half-initialised state outside SEH context. + if (_ocrDetector) _ocrDetector.reset(); + } + else if (!ocrSuccess) { + this->_logger.LogError("ANSALPR_OD::LoadEngine", + "Failed to load OCR detector (ONNX Runtime).", __FILE__, __LINE__); + if (_ocrDetector) _ocrDetector.reset(); + } } - // Check if we need to load the color model + // ── Step 8: LPC (optional) ────────────────────────── if (FileExist(colorModel) && (_lpColourModelConfig.detectionScoreThreshold > 0)) { + WriteEventLog("ANSALPR_OD::LoadEngine: Step 8 - Loading colour classifier with ONNX Runtime"); + this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 8: Loading colour classifier with ONNX Runtime", __FILE__, __LINE__); _lpColourModelConfig.detectionType = DetectionType::CLASSIFICATION; - _lpColourModelConfig.modelType = ModelType::ONNXYOLO; - _lpColourDetector = std::make_unique();// Classification with ONNX - bool colourSuccess = _lpColourDetector->LoadModelFromFolder(_licenseKey, _lpColourModelConfig, "lpc", "lpc.names", _modelFolder, _lpColourLabels); - if (!colourSuccess) { - this->_logger.LogError("ANSALPR_OD::LoadEngine", "Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__); - _lpColourDetector.reset(); - } - else { - _lpColourDetector->SetTracker(TrackerType::BYTETRACK, false); + _lpColourModelConfig.modelType = ModelType::ONNXYOLO; + { + LoadOnnxParams p{}; + p.licenseKey = &_licenseKey; + p.config = &_lpColourModelConfig; + p.modelFolder = &_modelFolder; + p.modelName = "lpc"; + p.classFile = "lpc.names"; + p.labels = &_lpColourLabels; + p.detector = &_lpColourDetector; + p.enableTracker = false; + p.disableStabilization = false; + + DWORD sehCode = 0; + bool colourSuccess = LoadOnnxModel_SEH(p, &sehCode); + if (sehCode != 0) { + char buf[256]; + snprintf(buf, sizeof(buf), + "ANSALPR_OD::LoadEngine: Step 8 LPC SEH exception 0x%08X — colour detection disabled", sehCode); + WriteEventLog(buf, EVENTLOG_ERROR_TYPE); + this->_logger.LogError("ANSALPR_OD::LoadEngine", + "Step 8: Colour classifier crashed (SEH 0x" + std::to_string(sehCode) + "). Colour detection disabled.", + __FILE__, __LINE__); + // Drop any half-initialised state outside SEH context. + if (_lpColourDetector) _lpColourDetector.reset(); + } + else if (!colourSuccess) { + this->_logger.LogError("ANSALPR_OD::LoadEngine", + "Failed to load colour detector (ONNX Runtime). Colour detection disabled.", __FILE__, __LINE__); + if (_lpColourDetector) _lpColourDetector.reset(); + } } } @@ -851,8 +1019,8 @@ namespace ANSCENTER { } } _isInitialized = valid; - WriteEventLog(("ANSALPR_OD::LoadEngine: Step 8 - Engine load complete. Valid = " + std::to_string(valid)).c_str()); - this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 8: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__); + WriteEventLog(("ANSALPR_OD::LoadEngine: Step 9 - Engine load complete. Valid = " + std::to_string(valid)).c_str()); + this->_logger.LogInfo("ANSALPR_OD::LoadEngine", "Step 9: Engine load complete. Valid = " + std::to_string(valid), __FILE__, __LINE__); return valid; } @@ -1467,8 +1635,11 @@ namespace ANSCENTER { constexpr int padding = 10; // --- Compute display→full-res scale (once per frame, cheap) --- + // NV12 GPU fast path is NVIDIA-only — cv::cuda::Stream/GpuMat + // touch the CUDA runtime even when the helper would early-return, + // which destabilises AMD/Intel hardware. Gate strictly on NVIDIA. float scaleX = 1.f, scaleY = 1.f; - { + if (isNvidiaEngine()) { auto* gpuData = tl_currentGpuFrame(); if (gpuData && gpuData->width > frame.cols && gpuData->height > frame.rows) { scaleX = static_cast(gpuData->width) / frame.cols; @@ -1484,8 +1655,9 @@ namespace ANSCENTER { cv::Mat lprImage; - // Try GPU NV12 crop (NVIDIA decode: NV12 still in GPU VRAM) - if (scaleX > 1.f) { + // Try GPU NV12 crop (NVIDIA decode: NV12 still in GPU VRAM). + // Skipped on AMD/Intel/CPU — see isNvidiaEngine() guard above. + if (isNvidiaEngine() && scaleX > 1.f) { auto cropResult = _nv12Helper.tryNV12CropToBGR( frame, 0, box, padding, scaleX, scaleY, this->_logger, "LPR"); @@ -1635,8 +1807,11 @@ namespace ANSCENTER { constexpr int padding = 10; // --- Compute display→full-res scale (once per frame, cheap) --- + // NVIDIA-only NV12 fast path — see isNvidiaEngine() discussion + // above. cv::cuda::* types touch CUDA even inside the "guarded" + // helper, so we must not even read tl_currentGpuFrame() on AMD. float scaleX2 = 1.f, scaleY2 = 1.f; - { + if (isNvidiaEngine()) { auto* gpuData = tl_currentGpuFrame(); if (gpuData && gpuData->width > frame.cols && gpuData->height > frame.rows) { scaleX2 = static_cast(gpuData->width) / frame.cols; @@ -1694,9 +1869,11 @@ namespace ANSCENTER { lprObject.cameraId = cameraId; lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows); - // Crop from full-res NV12 on GPU if available, otherwise display-res + // Crop from full-res NV12 on GPU if available, otherwise display-res. + // NV12 helper is NVIDIA-only — isNvidiaEngine() gate keeps + // CUDA runtime inactive on AMD/Intel/CPU hardware. cv::Mat lprImage; - if (scaleX2 > 1.f) { + if (isNvidiaEngine() && scaleX2 > 1.f) { auto cropResult = _nv12Helper.tryNV12CropToBGR( frame, 0, lprObject.box, 0, scaleX2, scaleY2, this->_logger, "LPR"); @@ -1760,10 +1937,12 @@ namespace ANSCENTER { lprObject.cameraId = cameraId; lprObject.polygon = RectToNormalizedPolygon(lprObject.box, input.cols, input.rows); - // Crop from full-res NV12 on GPU if available, otherwise display-res + // Crop from full-res NV12 on GPU if available, otherwise display-res. + // NV12 helper is NVIDIA-only — isNvidiaEngine() gate keeps + // CUDA runtime inactive on AMD/Intel/CPU hardware. cv::Rect lprPos(x1, y1, width, height); cv::Mat lprImage; - if (scaleX2 > 1.f) { + if (isNvidiaEngine() && scaleX2 > 1.f) { auto cropResult = _nv12Helper.tryNV12CropToBGR( frame, 0, lprPos, 0, scaleX2, scaleY2, this->_logger, "LPR"); diff --git a/modules/ANSLPR/ANSLPR_OD.h b/modules/ANSLPR/ANSLPR_OD.h index a436ee0..f9974fd 100644 --- a/modules/ANSLPR/ANSLPR_OD.h +++ b/modules/ANSLPR/ANSLPR_OD.h @@ -17,6 +17,30 @@ namespace ANSCENTER class ANSLPR_API ANSALPR_OD :public ANSALPR { private: ANSCENTER::EngineType engineType; + + // -------------------------------------------------------------- + // Vendor predicates — use these to gate hardware-specific paths + // (NV12 GPU crop, CUDA helpers, DirectML quirks, OpenVINO tricks). + // + // Certain helpers like _nv12Helper.tryNV12CropToBGR() call into + // CUDA runtime (cv::cuda::Stream/GpuMat/cudaStream_t) unconditionally, + // which is unsafe on AMD/Intel hardware — cv::cuda::Stream's ctor + // touches the CUDA driver even when the helper would early-return. + // Always wrap those calls in isNvidiaEngine() before invoking. + // -------------------------------------------------------------- + [[nodiscard]] bool isNvidiaEngine() const noexcept { + return engineType == ANSCENTER::EngineType::NVIDIA_GPU; + } + [[nodiscard]] bool isAmdEngine() const noexcept { + return engineType == ANSCENTER::EngineType::AMD_GPU; + } + [[nodiscard]] bool isIntelEngine() const noexcept { + return engineType == ANSCENTER::EngineType::OPENVINO_GPU; + } + [[nodiscard]] bool isCpuEngine() const noexcept { + return engineType == ANSCENTER::EngineType::CPU; + } + std::unique_ptr_lpDetector = nullptr; // License plate detector std::unique_ptr_ocrDetector = nullptr; // OCR detector std::unique_ptr_lpColourDetector = nullptr; // License plate colour classifier diff --git a/modules/ANSLPR/ANSLPR_OV.h b/modules/ANSLPR/ANSLPR_OV.h deleted file mode 100644 index 643acd6..0000000 --- a/modules/ANSLPR/ANSLPR_OV.h +++ /dev/null @@ -1,328 +0,0 @@ -#ifndef ANSLPROV_H -#define ANSLPROV_H -#pragma once -#include "ANSLPR.h" -#include -#include -#include -#include -#include - -#include "openvino/openvino.hpp" -#include "openvino/runtime/intel_gpu/properties.hpp" -#include "utils/args_helper.hpp" -#include "utils/input_wrappers.hpp" -#include "utils/common.hpp" -#include "utils/ocv_common.hpp" -#include "utils/threads_common.hpp" -#include "utils/grid_mat.hpp" -#include "monitors/presenter.h" - -namespace ANSCENTER -{ - void tryPush(const std::weak_ptr& worker, std::shared_ptr&& task); - - struct BboxAndDescr { - enum class ObjectType { - NONE, - VEHICLE, - PLATE, - } objectType; - cv::Rect rect; - std::string descr; - }; - - class Detector { - public: - struct Result { - std::size_t label; - float confidence; - cv::Rect location; - }; - static constexpr int maxProposalCount = 200; - static constexpr int objectSize = 7; // Output should have 7 as a last dimension" - Detector() = default; - Detector(ov::Core& core, - const std::string& deviceName, - const std::string& xmlPath, - const std::vector& detectionTresholds, - const bool autoResize); - ov::InferRequest createInferRequest(); - void setImage(ov::InferRequest& inferRequest, const cv::Mat& img); - std::list getResults(ov::InferRequest& inferRequest, cv::Size upscale, std::vector& rawResults); - - private: - bool m_autoResize; - std::vector m_detectionTresholds; - std::string m_detectorInputName; - std::string m_detectorOutputName; - ov::CompiledModel m_compiled_model; - }; - - class VehicleAttributesClassifier { - public: - VehicleAttributesClassifier() = default; - VehicleAttributesClassifier(ov::Core& core, - const std::string& deviceName, - const std::string& xmlPath, - const bool autoResize); - ov::InferRequest createInferRequest(); - void setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect vehicleRect); - std::pair getResults(ov::InferRequest& inferRequest); - private: - bool m_autoResize; - std::string m_attributesInputName; - std::string m_outputNameForColor; - std::string m_outputNameForType; - ov::CompiledModel m_compiled_model; - }; - - class Lpr { - public: - Lpr() = default; - Lpr(ov::Core& core, - const std::string& deviceName, - const std::string& xmlPath, - const bool autoResize); - ov::InferRequest createInferRequest(); - void setImage(ov::InferRequest& inferRequest, const cv::Mat& img, const cv::Rect plateRect); - std::string getResults(ov::InferRequest& inferRequest); - - private: - bool m_autoResize; - int m_maxSequenceSizePerPlate = 0; - std::string m_LprInputName; - std::string m_LprInputSeqName; - std::string m_LprOutputName; - ov::Layout m_modelLayout; - ov::CompiledModel m_compiled_model; - }; - - //Utilities - struct InferRequestsContainer { - InferRequestsContainer() = default; - InferRequestsContainer(const InferRequestsContainer&) = delete; - InferRequestsContainer& operator=(const InferRequestsContainer&) = delete; - - void assign(const std::vector& inferRequests) { - actualInferRequests = inferRequests; - this->inferRequests.container.clear(); - for (auto& ir : this->actualInferRequests) { - this->inferRequests.container.push_back(ir); - } - } - ConcurrentContainer>> inferRequests; - std::vector actualInferRequests; - }; - - - // stores all global data for tasks - struct Context { - Context(const std::vector>& inputChannels, - const Detector& detector, - const VehicleAttributesClassifier& vehicleAttributesClassifier, - const Lpr& lpr, - uint64_t lastFrameId, - uint64_t nireq, - bool isVideo, - std::size_t nclassifiersireq, - std::size_t nrecognizersireq) : - readersContext{ inputChannels, std::vector(inputChannels.size(), -1), std::vector(inputChannels.size()) }, - inferTasksContext{ detector }, - detectionsProcessorsContext{ vehicleAttributesClassifier, lpr }, - videoFramesContext{ std::vector(inputChannels.size(), lastFrameId), std::vector(inputChannels.size()) }, - nireq{ nireq }, - isVideo{ isVideo }, - freeDetectionInfersCount{ 0 }, - frameCounter{ 0 } - { - std::vector detectorInferRequests; - std::vector attributesInferRequests; - std::vector lprInferRequests; - detectorInferRequests.reserve(nireq); - attributesInferRequests.reserve(nclassifiersireq); - lprInferRequests.reserve(nrecognizersireq); - std::generate_n(std::back_inserter(detectorInferRequests), nireq, [&] {return inferTasksContext.detector.createInferRequest(); }); - std::generate_n(std::back_inserter(attributesInferRequests), nclassifiersireq, [&] {return detectionsProcessorsContext.vehicleAttributesClassifier.createInferRequest(); }); - std::generate_n(std::back_inserter(lprInferRequests), nrecognizersireq, [&] {return detectionsProcessorsContext.lpr.createInferRequest(); }); - detectorsInfers.assign(detectorInferRequests); - attributesInfers.assign(attributesInferRequests); - platesInfers.assign(lprInferRequests); - } - struct { - std::vector> inputChannels; - std::vector lastCapturedFrameIds; - std::vector lastCapturedFrameIdsMutexes; - std::weak_ptr readersWorker; - } readersContext; - - struct { - Detector detector; - std::weak_ptr inferTasksWorker; - } inferTasksContext; - - struct { - VehicleAttributesClassifier vehicleAttributesClassifier; - Lpr lpr; - std::weak_ptr detectionsProcessorsWorker; - } detectionsProcessorsContext; - struct { - std::vector lastframeIds; - std::vector lastFrameIdsMutexes; - } videoFramesContext; - - std::weak_ptr resAggregatorsWorker; - std::mutex classifiersAggregatorPrintMutex; - uint64_t nireq; - bool isVideo; - std::atomic::size_type> freeDetectionInfersCount; - std::atomic frameCounter; - InferRequestsContainer detectorsInfers, attributesInfers, platesInfers; - PerformanceMetrics metrics; - std::list boxesAndDescrs; - }; - - - - - // End of Context - class ReborningVideoFrame : public VideoFrame { - public: - ReborningVideoFrame(Context& context, - const unsigned sourceID, - const int64_t frameId, - const cv::Mat& frame = cv::Mat()) : - VideoFrame{ sourceID, frameId, frame }, - context(context) {} - virtual ~ReborningVideoFrame(); - Context& context; - }; - - // draws results on the frame - class ResAggregator : public Task { - public: - ResAggregator(const VideoFrame::Ptr& sharedVideoFrame, - std::list&& boxesAndDescrs) : - Task{ sharedVideoFrame, 4.0 }, - boxesAndDescrs{ std::move(boxesAndDescrs) } {} - - bool isReady() override { - return true; - } - void process() override; - private: - std::list boxesAndDescrs; - }; - - // waits for all classifiers and recognisers accumulating results - class ClassifiersAggregator { - public: - std::vector rawDetections; - ConcurrentContainer> rawAttributes; - ConcurrentContainer> rawDecodedPlates; - - explicit ClassifiersAggregator(const VideoFrame::Ptr& sharedVideoFrame) : - sharedVideoFrame{ sharedVideoFrame } {} - ~ClassifiersAggregator() { - std::mutex& printMutex = static_cast(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex; - printMutex.lock(); - if (!rawDetections.empty()) { - slog::debug << "Frame #: " << sharedVideoFrame->frameId << slog::endl; - slog::debug << rawDetections; - // destructor assures that none uses the container - for (const std::string& rawAttribute : rawAttributes.container) { - slog::debug << rawAttribute << slog::endl; - } - for (const std::string& rawDecodedPlate : rawDecodedPlates.container) { - slog::debug << rawDecodedPlate << slog::endl; - } - } - printMutex.unlock(); - tryPush(static_cast(sharedVideoFrame.get())->context.resAggregatorsWorker, - std::make_shared(sharedVideoFrame, std::move(boxesAndDescrs))); - } - - void push(BboxAndDescr&& bboxAndDescr) { - boxesAndDescrs.lockedPushBack(std::move(bboxAndDescr)); - } - const VideoFrame::Ptr sharedVideoFrame; - - private: - ConcurrentContainer> boxesAndDescrs; - }; - - // extracts detections from blob InferRequests and runs classifiers and recognisers - class DetectionsProcessor : public Task { - public: - DetectionsProcessor(VideoFrame::Ptr sharedVideoFrame, ov::InferRequest* inferRequest) : - Task{ sharedVideoFrame, 1.0 }, - inferRequest{ inferRequest }, - requireGettingNumberOfDetections{ true } - { - - } - - DetectionsProcessor(VideoFrame::Ptr sharedVideoFrame, - std::shared_ptr&& classifiersAggregator, - std::list&& vehicleRects, - std::list&& plateRects) : - Task{ sharedVideoFrame, 1.0 }, - classifiersAggregator{ std::move(classifiersAggregator) }, - inferRequest{ nullptr }, - vehicleRects{ std::move(vehicleRects) }, - plateRects{ std::move(plateRects) }, - requireGettingNumberOfDetections{ false } - { - - } - - bool isReady() override; - void process() override; - - private: - std::shared_ptr classifiersAggregator; // when no one stores this object we will draw - ov::InferRequest* inferRequest; - std::list vehicleRects; - std::list plateRects; - std::vector> reservedAttributesRequests; - std::vector> reservedLprRequests; - bool requireGettingNumberOfDetections; - - }; - // runs detection - class InferTask : public Task { - public: - explicit InferTask(VideoFrame::Ptr sharedVideoFrame) : - Task{ sharedVideoFrame, 5.0 } {} - bool isReady() override; - void process() override; - }; - - class Reader : public Task { - public: - explicit Reader(VideoFrame::Ptr sharedVideoFrame) : - Task{ sharedVideoFrame, 2.0 } {} - bool isReady() override; - void process() override; - }; - - - class ANSLPR_API ANSALPR_OV :public ANSALPR { - private: - std::string _vehicleLPModel; //model that detects both vehicle and license plate - std::string _vehicleAtModel; //model that detects vehicle attributes - std::string _lprModel; //model that recognise license plate - Detector* _detector = nullptr; - VehicleAttributesClassifier* _vehicleAttributesClassifier = nullptr; - Lpr* _lpr = nullptr; - [[nodiscard]] std::string VectorDetectionToJsonString(const std::vector& dets); - public: - ANSALPR_OV(); - ~ANSALPR_OV(); - [[nodiscard]] bool Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) override; - [[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult); - [[nodiscard]] bool Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult); - [[nodiscard]] bool Destroy() override; - }; -} -#endif \ No newline at end of file diff --git a/modules/ANSLPR/ANSLPR_RT.cpp b/modules/ANSLPR/ANSLPR_RT.cpp deleted file mode 100644 index 1f0796a..0000000 --- a/modules/ANSLPR/ANSLPR_RT.cpp +++ /dev/null @@ -1,776 +0,0 @@ -#include "ANSLPR_RT.h" - -namespace ANSCENTER { - ANSALPR_RT::ANSALPR_RT() { - _licenseValid = false; - _globalViewId = 0; - _focusedOnLPRId = 0; - _platesTypesClassifierId = 0; - } - ANSALPR_RT::~ANSALPR_RT() { - CloseReferences(); - } - bool ANSALPR_RT::Destroy() { - CloseReferences(); - return true; - } - bool ANSALPR_RT::CloseReferences() { - try { - bool session_closed = CloseDetector(_globalViewId); - if (!session_closed) { - this->_logger->LogFatal("ANSALPR_RT::CloseReferences", "Cannot close global view model reference", __FILE__, __LINE__); - } - else { - _globalViewId = 0; - } - session_closed = CloseDetector(_focusedOnLPRId); - if (!session_closed) { - this->_logger->LogFatal("ANSALPR_RT::CloseReferences", "Cannot close focused on LRP model reference", __FILE__, __LINE__); - } - else { - _focusedOnLPRId = 0; - } - - session_closed = ClosePlatesTypesClassifier(_platesTypesClassifierId); - if (!session_closed) { - this->_logger->LogFatal("ANSALPR_RT::CloseReferences", "Cannot close plate type classifer model reference", __FILE__, __LINE__); - } - else { - _platesTypesClassifierId = 0; - } - return true; - } - catch (std::exception& e) { - this->_logger->LogFatal("ANSALPR_RT::CloseReferences", e.what(), __FILE__, __LINE__); - return false; - } - } - bool ANSALPR_RT::Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) { - try { - _licenseKey = licenseKey; - _licenseValid = false; - CheckLicense(); - if (!_licenseValid) { - this->_logger->LogError("ANSALPR::Initialize.", "License is not valid.", __FILE__, __LINE__); - return false; - } - // Extract model folder - // 0. Check if the modelZipFilePath exist? - if (!FileExist(modelZipFilePath)) { - this->_logger->LogFatal("ANSALPR::Initialize", "Model zip file is not exist", __FILE__, __LINE__); - } - // 1. Unzip model zip file to a special location with folder name as model file (and version) - std::string outputFolder; - std::vector passwordArray; - if (!modelZipPassword.empty()) passwordArray.push_back(modelZipPassword); - passwordArray.push_back("AnsDemoModels20@!"); - passwordArray.push_back("Sh7O7nUe7vJ/417W0gWX+dSdfcP9hUqtf/fEqJGqxYL3PedvHubJag=="); - passwordArray.push_back("3LHxGrjQ7kKDJBD9MX86H96mtKLJaZcTYXrYRdQgW8BKGt7enZHYMg=="); - std::string modelName = GetFileNameWithoutExtension(modelZipFilePath); - //this->_logger->LogInfo("ANSFDBase::Initialize. Model name", modelName); - - size_t vectorSize = passwordArray.size(); - for (size_t i = 0; i < vectorSize; i++) { - if (ExtractPasswordProtectedZip(modelZipFilePath, passwordArray[i], modelName, _modelFolder, false)) - break; // Break the loop when the condition is met. - } - // 2. Check if the outputFolder exist - if (!FolderExist(_modelFolder)) { - this->_logger->LogError("ANSFDBase::Initialize. Output model folder is not exist", _modelFolder, __FILE__, __LINE__); - return false; // That means the model file is not exist or the password is not correct - } - - - //3. Get License plate models - _globalModelFileName = CreateFilePath(_modelFolder, "anslpr_alpr_focused_on_lp.onnx"); - if (!FileExist(_globalModelFileName)) { - this->_logger->LogError("ANSALPR::Initialize. Global view model does not exist", _globalModelFileName, __FILE__, __LINE__); - return false; - } - _focusedOnLPRModelFileName = CreateFilePath(_modelFolder, "anslpr_alpr_global_view.onnx"); - if (!FileExist(_focusedOnLPRModelFileName)) { - this->_logger->LogError("ANSALPR::Initialize. Focused On LRP model does not exist", _focusedOnLPRModelFileName, __FILE__, __LINE__); - return false; - } - _platesTypesClassifierFileName = CreateFilePath(_modelFolder, "plates_types_7.onnx"); - if (!FileExist(_platesTypesClassifierFileName)) { - this->_logger->LogError("ANSALPR::Initialize.LRP classifier model does not exist", _platesTypesClassifierFileName, __FILE__, __LINE__); - return false; - } - - _platesTypesLabelsFileName = CreateFilePath(_modelFolder, "plates_types_7.txt"); - if (!FileExist(_platesTypesLabelsFileName)) { - this->_logger->LogError("ANSALPR::Initialize.LRP classifier label model does not exist", _platesTypesLabelsFileName, __FILE__, __LINE__); - return false; - } - - // Load models - size_t len = _globalModelFileName.size(); - - //step 1 : Initializes a new detector by loading its model file. In return, you get a unique id. The repo comes with two models namely anslpr_alpr_focused_on_lpand anslpr_alpr_global_view. - //So you have to call this function twice to initialize both models. - - _globalViewId = InitYoloDetector(len, _globalModelFileName.c_str()); - if (_globalViewId <= 0) { - this->_logger->LogError("ANSALPR::Initialize.", "Global view model cannot be loaded", __FILE__, __LINE__); - return false; - } - - len = _focusedOnLPRModelFileName.size(); - _focusedOnLPRId = InitYoloDetector(len, _focusedOnLPRModelFileName.c_str()); - - if (_focusedOnLPRId <= 0) { - this->_logger->LogError("ANSALPR::Initialize.", "Focused on LPR model cannot be loaded", __FILE__, __LINE__); - return false; - } - - len = _platesTypesClassifierFileName.size(); - _platesTypesClassifierId = InitPlatesClassifer(len, _platesTypesClassifierFileName.c_str(), _platesTypesLabelsFileName.size(), _platesTypesLabelsFileName.c_str()); - - if (_platesTypesClassifierId <= 0) { - this->_logger->LogError("ANSALPR::Initialize.", "Plate type classifier model cannot be loaded", __FILE__, __LINE__); - return false; - } - return true; - } - catch (std::exception& e) { - this->_logger->LogFatal("ANSALPR::Initialize", e.what(), __FILE__, __LINE__); - return false; - } - } - bool ANSALPR_RT::Inference(const cv::Mat& input, std::string& lprResult) - { - if (!_licenseValid) { - lprResult = ""; - return false; - } - try { - const size_t lpn_len = 15; - char* lpn = new char[lpn_len + 1]; - std::vector output; - output.clear(); - cv::Rect bbox; - bool detected = TwoStageLPRPlatesTypeDetection( - input.cols,//width of image - input.rows,//height of image i.e. the specified dimensions of the image - input.channels(),// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - input.data, - input.step,// source image bytes buffer - _globalViewId, - _focusedOnLPRId,//id : unique interger to identify the detector to be used - _platesTypesClassifierId,//unique id to identify the platestype classifier - lpn_len, - lpn, bbox); - lprResult = lpn; - if (detected) { - ALPRObject result; - result.classId = 0; - result.className = lpn; - result.confidence = 1.0; - result.box = bbox; - output.push_back(result); - } - lprResult = VectorDetectionToJsonString(output); - return detected; - } - catch (std::exception& e) { - this->_logger->LogFatal("ANSALPR::CheckStatus", e.what(), __FILE__, __LINE__); - lprResult = ""; - return false; - }; - } - bool ANSALPR_RT::Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult) { - if (!_licenseValid) { - lprResult = ""; - return false; - } - try { - const size_t lpn_len = 15; - char* lpn = new char[lpn_len + 1]; - std::vector output; - output.clear(); - bool detected = false; - if (Bbox.size() > 0) { - cv::Mat frame = input.clone(); - for (std::vector::iterator it = Bbox.begin(); it != Bbox.end(); it++) { - int x1, y1, x2, y2; - x1 = (*it).x; - y1 = (*it).y; - x2 = (*it).x + (*it).width; - y2 = (*it).y + (*it).height; - - // Get cropped objects - cv::Rect objectPos(cv::Point(x1, y1), cv::Point(x2, y2)); - cv::Mat croppedObject = frame(objectPos); - - cv::Rect bbox; - detected = TwoStageLPRPlatesTypeDetection( - croppedObject.cols,//width of image - croppedObject.rows,//height of image i.e. the specified dimensions of the image - croppedObject.channels(),// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - croppedObject.data, - croppedObject.step,// source image bytes buffer - _globalViewId, - _focusedOnLPRId,//id : unique interger to identify the detector to be used - _platesTypesClassifierId,//unique id to identify the platestype classifier - lpn_len, - lpn, bbox); - lprResult = lpn; - if (detected) { - ALPRObject result; - result.classId = 0; - result.className = lpn; - result.confidence = 1.0; - result.box.x = bbox.x + x1; - result.box.y = bbox.y + y1; - result.box.width = bbox.width; - result.box.height = bbox.height; - output.push_back(result); - } - } - lprResult = VectorDetectionToJsonString(output); - } - else { - cv::Rect bbox; - detected = TwoStageLPRPlatesTypeDetection( - input.cols,//width of image - input.rows,//height of image i.e. the specified dimensions of the image - input.channels(),// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - input.data, - input.step,// source image bytes buffer - _globalViewId, - _focusedOnLPRId,//id : unique interger to identify the detector to be used - _platesTypesClassifierId,//unique id to identify the platestype classifier - lpn_len, - lpn, bbox); - lprResult = lpn; - if (detected) { - ALPRObject result; - result.classId = 0; - result.className = lpn; - result.confidence = 1.0; - result.box = bbox; - output.push_back(result); - } - lprResult = VectorDetectionToJsonString(output); - } - return detected; - } - catch (std::exception& e) { - this->_logger->LogFatal("ANSALPR::CheckStatus", e.what(), __FILE__, __LINE__); - lprResult = ""; - return false; - }; - } - - // Private: - std::string ANSALPR_RT::VectorDetectionToJsonString(const std::vector& dets) { - boost::property_tree::ptree root; - boost::property_tree::ptree detectedObjects; - for (int i = 0; i < dets.size(); i++) { - boost::property_tree::ptree detectedNode; - detectedNode.put("class_id", dets[i].classId); - detectedNode.put("class_name", dets[i].className); - detectedNode.put("prob", dets[i].confidence); - detectedNode.put("x", dets[i].box.x); - detectedNode.put("y", dets[i].box.y); - detectedNode.put("width", dets[i].box.width); - detectedNode.put("height", dets[i].box.height); - detectedNode.put("mask", "");//Todo: convert masks to mask with comma seperated dets[i].mask); - detectedNode.put("extra_info", ""); - - // we might add masks into this using comma seperated string - detectedObjects.push_back(std::make_pair("", detectedNode)); - } - root.add_child("results", detectedObjects); - std::ostringstream stream; - boost::property_tree::write_json(stream, root, false); - std::string trackingResult = stream.str(); - return trackingResult; - } - - - - void ANSALPR_RT::CheckStatus(OrtStatus* status) { - try { - if (status != nullptr) { - const char* msg = g_ort->GetErrorMessage(status); - this->_logger->LogError("ANSALPR::CheckStatus", msg, __FILE__, __LINE__); - g_ort->ReleaseStatus(status); - exit(1); - } - } - catch (std::exception& e) { - this->_logger->LogFatal("ANSALPR::CheckStatus", e.what(), __FILE__, __LINE__); - } - } - - std::list::const_iterator ANSALPR_RT::GetDetector(unsigned int id, const std::list& detectors, - const std::list& detectors_ids) - { - assert(detectors_ids.size() == detectors.size()); - std::list::const_iterator it(detectors.begin()); - std::list::const_iterator it_id(detectors_ids.begin()); - while (it != detectors.end() && it_id != detectors_ids.end()) { - if (*it_id == id) { - return it; - } - else { - it_id++; - it++; - } - } - return detectors.end(); - } - - std::list::const_iterator ANSALPR_RT::GetPlatesTypesClassifier(unsigned int id, const std::list& plates_types_classifiers, - const std::list& plates_types_classifiers_ids) { - assert(plates_types_classifiers_ids.size() == plates_types_classifiers.size()); - std::list::const_iterator it(plates_types_classifiers.begin()); - std::list::const_iterator it_id(plates_types_classifiers_ids.begin()); - while (it != plates_types_classifiers.end() && it_id != plates_types_classifiers_ids.end()) { - if (*it_id == id) { - return it; - } - else { - it_id++; - it++; - } - } - return plates_types_classifiers.end(); - } - - unsigned int ANSALPR_RT::GetNewId(const std::list& detectors_ids) { - if (detectors_ids.size()) { - auto result = std::minmax_element(detectors_ids.begin(), detectors_ids.end()); - return *result.second + 1; - } - else return 1; - } - bool ANSALPR_RT::CloseDetector(unsigned int id, std::list& _envs, std::list& _lsessionOptions, std::list& _detectors, - std::list& _detectors_ids) { - assert(_detectors_ids.size() == _detectors.size() - && _detectors_ids.size() == _envs.size() - && _detectors_ids.size() == _lsessionOptions.size()); - std::list::iterator it(_detectors.begin()); - std::list::iterator it_id(_detectors_ids.begin()); - std::list::iterator it_sessionOptions(_lsessionOptions.begin()); - std::list::iterator it_envs(_envs.begin()); - while (it != _detectors.end() && it_id != _detectors_ids.end() - && it_envs != _envs.end() && it_sessionOptions != _lsessionOptions.end() - ) { - if (*it_id == id) { - if (*it != nullptr) delete* it; - if (*it_sessionOptions != nullptr) delete* it_sessionOptions; - if (*it_envs != nullptr) delete* it_envs; - it_envs = _envs.erase(it_envs); - it_sessionOptions = _lsessionOptions.erase(it_sessionOptions); - it = _detectors.erase(it); - it_id = _detectors_ids.erase(it_id); - return true; - } - else { - it_sessionOptions++; - it_envs++; - it_id++; - it++; - } - } - return false; - } - bool ANSALPR_RT::CloseDetector(unsigned int id, std::list& _envs, std::list& _lsessionOptions, std::list& _detectors, - std::list& _detectors_ids) { - assert(_detectors_ids.size() == _detectors.size() - && _detectors_ids.size() == _envs.size() - && _detectors_ids.size() == _lsessionOptions.size()); - std::list::iterator it(_detectors.begin()); - std::list::iterator it_id(_detectors_ids.begin()); - std::list::iterator it_sessionOptions(_lsessionOptions.begin()); - std::list::iterator it_envs(_envs.begin()); - while (it != _detectors.end() && it_id != _detectors_ids.end() - && it_envs != _envs.end() && it_sessionOptions != _lsessionOptions.end() - ) { - if (*it_id == id) { - if (*it != nullptr) delete* it; - if (*it_sessionOptions != nullptr) delete* it_sessionOptions; - if (*it_envs != nullptr) delete* it_envs; - it_envs = _envs.erase(it_envs); - it_sessionOptions = _lsessionOptions.erase(it_sessionOptions); - it = _detectors.erase(it); - it_id = _detectors_ids.erase(it_id); - return true; - } - else { - it_sessionOptions++; - it_envs++; - it_id++; - it++; - } - } - return false; - } - - // Private interface - unsigned int ANSALPR_RT::InitYoloDetector(unsigned int len, const char* model_file) - { - assert(detectors_ids.size() == detectors.size()); - const std::string model_filename(model_file, len); - if (!model_filename.size() || !std::filesystem::exists(model_filename) - || !std::filesystem::is_regular_file(model_filename) - ) - { - this->_logger->LogError("ANSALPR::InitYoloDetector. Model file is not regular file.", model_filename, __FILE__, __LINE__); - return 0; - } - //step 2 declare an onnx runtime environment - std::string instanceName{ "image-classification-inference" }; - // https://github.com/microsoft/onnxruntime/blob/rel-1.6.0/include/onnxruntime/core/session/onnxruntime_c_api.h#L123 - Ort::Env* penv = new Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str()); - if (penv != nullptr) { - //step 3 declare options for the runtime environment - Ort::SessionOptions* psessionOptions = new Ort::SessionOptions(); - if (psessionOptions != nullptr) { - psessionOptions->SetIntraOpNumThreads(1); - // Sets graph optimization level - // Available levels are - // ORT_DISABLE_ALL -> To disable all optimizations - // ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node - // removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations - // (Includes level 1 + more complex optimizations like node fusions) - // ORT_ENABLE_ALL -> To Enable All possible optimizations - psessionOptions->SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); -#ifdef ANSLPR_USE_CUDA - // Optionally add more execution providers via session_options - // E.g. for CUDA include cuda_provider_factory.h and uncomment the following line: - // nullptr for Status* indicates success - OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(*psessionOptions, 0); - //or status =nullptr; //if you don t have CUDA - if (status == nullptr) { -#endif //ANSLPR_USE_CUDA - Yolov5_alpr_onxx_detector* onnx_net = nullptr; -#ifdef _WIN32 - //step 4 declare an onnx session (ie model), by giving references to the runtime environment, session options and file path to the model - std::wstring widestr = std::wstring(model_filename.begin(), model_filename.end()); - onnx_net = new Yolov5_alpr_onxx_detector(*penv, widestr.c_str(), *psessionOptions); -#else - onnx_net = new Yolov5_alpr_onxx_detector(*penv, model_filename.c_str(), *psessionOptions); -#endif - if (onnx_net != nullptr && penv != nullptr && psessionOptions != nullptr) { - std::unique_lock lck(mtxMutex, std::defer_lock); - lck.lock(); - detectors_envs.push_back(penv); - l_detectors_sessionOptions.push_back(psessionOptions); - detectors.push_back(onnx_net); - unsigned int id = GetNewId(detectors_ids); - detectors_ids.push_back(id); - lck.unlock(); - return id; - } - else { - this->_logger->LogError("ANSALPR::InitYoloDetector. Error while creating onnxruntime session with file", model_filename.c_str(), __FILE__, __LINE__); - return 0; - } -#ifdef ANSLPR_USE_CUDA - } - else { - CheckStatus(status); - this->_logger->LogError("ANSALPR::InitYoloDetector.", "Cuda error", __FILE__, __LINE__); - return 0; - } -#endif //ANSLPR_USE_CUDA - } - else { - this->_logger->LogError("ANSALPR::InitYoloDetector.", "Error while creating SessionOptions", __FILE__, __LINE__); - return 0; - } - } - else { - this->_logger->LogError("ANSALPR::InitYoloDetector.", "Error while creating while creating session environment (Ort::Env)", __FILE__, __LINE__); - return 0; - } - } - unsigned int ANSALPR_RT::InitPlatesClassifer(unsigned int len_models_filename, const char* model_file, unsigned int len_labels_filename, const char* labels_file) - { - assert(plates_types_classifier_ids.size() == plates_types_classifiers.size()); - const std::string model_filename(model_file, len_models_filename); - const std::string labels_filename(labels_file, len_labels_filename); - if (!model_filename.size() || !std::filesystem::exists(model_filename) - || !std::filesystem::is_regular_file(model_filename) - || !labels_filename.size() || !std::filesystem::exists(labels_filename) - || !std::filesystem::is_regular_file(labels_filename) - ) - { - this->_logger->LogDebug("ANSALPR::InitPlatesClassifer. Model file is not regular file.", model_filename, __FILE__, __LINE__); - return 0; - } - //step 2 declare an onnx runtime environment - std::string instanceName{ "image-classification-inference" }; - // https://github.com/microsoft/onnxruntime/blob/rel-1.6.0/include/onnxruntime/core/session/onnxruntime_c_api.h#L123 - Ort::Env* penv = new Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str()); - if (penv != nullptr) { - //step 3 declare options for the runtime environment - Ort::SessionOptions* psessionOptions = new Ort::SessionOptions(); - if (psessionOptions != nullptr) { - psessionOptions->SetIntraOpNumThreads(1); - // Sets graph optimization level - // Available levels are - // ORT_DISABLE_ALL -> To disable all optimizations - // ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node - // removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations - // (Includes level 1 + more complex optimizations like node fusions) - // ORT_ENABLE_ALL -> To Enable All possible optimizations - psessionOptions->SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); -#ifdef ANSLPR_USE_CUDA - // Optionally add more execution providers via session_options - // E.g. for CUDA include cuda_provider_factory.h and uncomment the following line: - // nullptr for Status* indicates success - OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(*psessionOptions, 0); - //or status =nullptr; //if you don t have CUDA - if (status == nullptr) { -#endif //ANSLPR_USE_CUDA - Plates_types_classifier* onnx_net = nullptr; -#ifdef _WIN32 - //step 4 declare an onnx session (ie model), by giving references to the runtime environment, session options and file path to the model - std::wstring widestr = std::wstring(model_filename.begin(), model_filename.end()); - onnx_net = new Plates_types_classifier(*penv, widestr.c_str(), *psessionOptions, labels_filename); -#else - onnx_net = new Plates_types_classifier(*penv, model_filename.c_str(), *psessionOptions, labels_filename); -#endif - if (onnx_net != nullptr && penv != nullptr && psessionOptions != nullptr) { - std::unique_lock lck(mtxMutex, std::defer_lock); - lck.lock(); - plates_types_envs.push_back(penv); - l_plates_types_classifier_sessionOptions.push_back(psessionOptions); - plates_types_classifiers.push_back(onnx_net); - unsigned int id = GetNewId(plates_types_classifier_ids); - plates_types_classifier_ids.push_back(id); - lck.unlock(); - return id; - } - else { - this->_logger->LogError("ANSALPR::InitPlatesClassifer. Error while creating onnxruntime session with file.", model_filename, __FILE__, __LINE__); - return 0; - } -#ifdef ANSLPR_USE_CUDA - } - else { - CheckStatus(status); - this->_logger->LogError("ANSALPR::InitPlatesClassifer.", "Cuda error", __FILE__, __LINE__); - return 0; - } -#endif //ANSLPR_USE_CUDA - } - else { - this->_logger->LogError("ANSALPR::InitPlatesClassifer.", "Error while creating SessionOptions", __FILE__, __LINE__); - return 0; - } - } - else { - this->_logger->LogError("ANSALPR::InitPlatesClassifer.", "Error while creating session environment (Ort::Env)", __FILE__, __LINE__); - return 0; - } - } - bool ANSALPR_RT::TwoStageALPR(const int width,//width of image - const int height,//height of image i.e. the specified dimensions of the image - const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - void* pbData, unsigned int step,// source image bytes buffer - unsigned int id_global_view, - unsigned int id_focused_on_lp, - unsigned int lpn_len, - char* lpn, cv::Rect& bbox) - { - if ((pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr) { - this->_logger->LogError("ANSALPR::TwoStageLPR.", "Condition on image (pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr not met", __FILE__, __LINE__); - return false; - } - else { - cv::Mat destMat; - if (pixOpt == 1) - { - destMat = cv::Mat(height, width, CV_8UC1, pbData, step); - } - if (pixOpt == 3) - { - destMat = cv::Mat(height, width, CV_8UC3, pbData, step); - } - if (pixOpt == 4) - { - destMat = cv::Mat(height, width, CV_8UC4, pbData, step); - } - std::list::const_iterator it_global_view = GetDetector(id_global_view, detectors, detectors_ids); - if (it_global_view != detectors.end()) { - std::list::const_iterator it_focused_on_lp = GetDetector(id_focused_on_lp, detectors, detectors_ids); - std::string lpn_str; - std::list ROIs; - - std::unique_lock lck(mtxMutex, std::defer_lock); - if (it_focused_on_lp != detectors.end()) { - lck.lock(); - //for normal plates - ROIs = (*it_global_view)->TwoStage_LPR(*(*it_focused_on_lp), destMat, lpn_str); - - //for small plates - lck.unlock(); - } - else { - this->_logger->LogError("ANSALPR::TwoStageLPR.", "id_focused_on_lp does not point to a valid detector", __FILE__, __LINE__); - lck.lock(); - //for normal plates - ROIs = (*it_global_view)->TwoStage_LPR(*(*it_global_view), destMat, lpn_str); - //for small plates - lck.unlock(); - } - std::string::const_iterator it_lpn(lpn_str.begin()); - int i = 0; - while (it_lpn != lpn_str.end() && i < lpn_len - 1) { - lpn[i] = *it_lpn; - i++; it_lpn++; - } - while (i < lpn_len) { - lpn[i] = '\0'; - i++; - } - bbox = GetGlobalROI(ROIs); - return (lpn_str.length() > 0); - } - else { - this->_logger->LogError("ANSALPR::TwoStageLPR.", "id_global_view does not point to a valid detector", __FILE__, __LINE__); - return false; - } - } - } - - cv::Rect ANSALPR_RT::GetGlobalROI(std::list ROIs) { - cv::Rect result; - if (ROIs.size() > 0) result = ROIs.front(); - return result; - - /*std::list::iterator it; - cv::Rect v = ROIs.front(); - int x_min, y_min, w_min, h_min; - int x_max, y_max, w_max, h_max; - x_min = v.x; - x_max = v.x; - y_min = v.y; - y_max = v.y; - w_min = v.width; - w_max = v.width; - h_min = v.height; - h_max = v.height; - for (it = ROIs.begin(); it != ROIs.end(); ++it) { - if (x_min > it->x) { x_min = it->x; } - if (x_max < it->x) { x_max = it->x; } - if (y_min > it->y) { y_min = it->y; } - if (y_max < it->y) { y_max = it->y; } - if (w_min > it->width) { w_min = it->width; } - if (w_max < it->width) { w_max = it->width; } - if (h_min > it->height) { w_min = it->height; } - if (h_max < it->height) { w_max = it->height; } - } - v.x = x_min - 2; - v.y = y_min - 2; - v.width = x_max + w_max - x_min + 2; - v.height = h_max + 2; - return v;*/ - } - - bool ANSALPR_RT::TwoStageLPRPlatesTypeDetection(const int width,//width of image - const int height,//height of image i.e. the specified dimensions of the image - const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - void* pbData, unsigned int step,// source image bytes buffer - unsigned int id_global_view, - unsigned int id_focused_on_lp, - unsigned int id_plates_types_classifier, - unsigned int lpn_len, char* lpn, cv::Rect& bbox) - { - if ((pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr) { - this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "Condition on image (pixOpt != 1) && (pixOpt != 3) && (pixOpt != 4) || height <= 0 || width <= 0 || pbData == nullptr not met", __FILE__, __LINE__); - return false; - } - else { - cv::Mat destMat; - if (pixOpt == 1) - { - destMat = cv::Mat(height, width, CV_8UC1, pbData, step); - } - if (pixOpt == 3) - { - destMat = cv::Mat(height, width, CV_8UC3, pbData, step); - } - if (pixOpt == 4) - { - destMat = cv::Mat(height, width, CV_8UC4, pbData, step); - } - std::list::const_iterator it_plates_types_classifier = GetPlatesTypesClassifier(id_plates_types_classifier, - plates_types_classifiers, plates_types_classifier_ids); - if (it_plates_types_classifier != plates_types_classifiers.end()) { - std::list::const_iterator it_global_view = GetDetector(id_global_view, detectors, detectors_ids); - if (it_global_view != detectors.end()) { - std::list::const_iterator it_focused_on_lp = GetDetector(id_focused_on_lp, detectors, detectors_ids); - std::string lpn_str; - std::list ROIs; - std::unique_lock lck(mtxMutex, std::defer_lock); - if (it_focused_on_lp != detectors.end()) { - lck.lock(); - //for normal plates - ROIs = (*it_global_view)->TwoStageLPR(*(*it_focused_on_lp), *(*it_plates_types_classifier), destMat, lpn_str); - //for small plates - lck.unlock(); - } - else { - this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "id_focused_on_lp does not point to a valid detector", __FILE__, __LINE__); - lck.lock(); - //for normal plates - ROIs = (*it_global_view)->TwoStageLPR(*(*it_global_view), *(*it_plates_types_classifier), destMat, lpn_str); - //for small plates - lck.unlock(); - } - std::string::const_iterator it_lpn(lpn_str.begin()); - int i = 0; - while (it_lpn != lpn_str.end() && i < lpn_len - 1) { - lpn[i] = *it_lpn; - i++; it_lpn++; - } - while (i < lpn_len) { - lpn[i] = '\0'; - i++; - } - - bbox = GetGlobalROI(ROIs); - return (lpn_str.length() > 0); - } - else { - this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "id_global_view does not point to a valid detector", __FILE__, __LINE__); - return false; - } - } - else { - this->_logger->LogError("ANSALPR::TwoStageLPRPlatesTypeDetection.", "id_plates_types_classifier does not point to a valid detector", __FILE__, __LINE__); - return TwoStageALPR(width,//width of image - height,//height of image i.e. the specified dimensions of the image - pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - pbData, step,// source image bytes buffer - id_global_view, id_focused_on_lp, lpn_len, lpn, bbox); - } - } - } - - bool ANSALPR_RT::CloseDetector(unsigned int id) - { - assert(detectors_ids.size() == detectors.size()); - std::unique_lock lck(mtxMutex, std::defer_lock); - lck.lock(); - bool session_closed = CloseDetector(id, detectors_envs, l_detectors_sessionOptions, detectors, detectors_ids); - lck.unlock(); - return session_closed; - } - bool ANSALPR_RT::ClosePlatesTypesClassifier(unsigned int id) - { - assert(plates_types_classifier_ids.size() == plates_types_classifiers.size()); - std::unique_lock lck(mtxMutex, std::defer_lock); - lck.lock(); - bool session_closed = CloseDetector(id, plates_types_envs, l_plates_types_classifier_sessionOptions, plates_types_classifiers, plates_types_classifier_ids); - lck.unlock(); - return session_closed; - } -} diff --git a/modules/ANSLPR/ANSLPR_RT.h b/modules/ANSLPR/ANSLPR_RT.h deleted file mode 100644 index 07ca3bd..0000000 --- a/modules/ANSLPR/ANSLPR_RT.h +++ /dev/null @@ -1,97 +0,0 @@ -#ifndef ANSLPRRT_H -#define ANSLPRRT_H -#pragma once -#include "ANSLPR.h" -#include -#include -#include "yolov5_alpr_onnx_detector.h" -#include "ONNX_detector.h" -#include // std::thread -#include // std::mutex, std::unique_lock, std::defer_lock - -namespace ANSCENTER -{ - class ANSLPR_API ANSALPR_RT :public ANSALPR { - private: - std::mutex mtxMutex; // mutex for critical section - std::string _globalModelFileName{}; - std::string _focusedOnLPRModelFileName{}; - std::string _platesTypesClassifierFileName{}; - std::string _platesTypesLabelsFileName{}; - - size_t _globalViewId; - size_t _focusedOnLPRId; - size_t _platesTypesClassifierId; - [[nodiscard]] bool CloseReferences(); - - public: - ANSALPR_RT(); - ~ANSALPR_RT(); - [[nodiscard]] bool Initialize(const std::string& licenseKey, const std::string& modelZipFilePath, const std::string& modelZipPassword) override; - [[nodiscard]] bool Inference(const cv::Mat& input, std::string& lprResult); - [[nodiscard]] bool Inference(const cv::Mat& input, const std::vector& Bbox, std::string& lprResult); - [[nodiscard]] bool Destroy() override; - - private: - [[nodiscard]] std::string VectorDetectionToJsonString(const std::vector& dets); - void CheckStatus(OrtStatus* status); - cv::Rect GetGlobalROI(std::list ROIs); - std::list::const_iterator GetDetector(unsigned int id, const std::list& detectors, - const std::list& detectors_ids); - std::list::const_iterator GetPlatesTypesClassifier(unsigned int id, const std::list& plates_types_classifiers, - const std::list& plates_types_classifiers_ids); - unsigned int GetNewId(const std::list& detectors_ids); - bool CloseDetector(unsigned int id, std::list& _envs, std::list& _lsessionOptions, std::list& _detectors, - std::list& _detectors_ids); - bool CloseDetector(unsigned int id, std::list& _envs, std::list& _lsessionOptions, std::list& _detectors, - std::list& _detectors_ids); - - unsigned int InitYoloDetector(unsigned int len, const char* model_file); - unsigned int InitPlatesClassifer(unsigned int len_models_filename, const char* model_file, unsigned int len_labels_filename, const char* labels_file); - bool TwoStageALPR(const int width,//width of image - const int height,//height of image i.e. the specified dimensions of the image - const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - void* pbData, unsigned int step,// source image bytes buffer - unsigned int id_global_view, - unsigned int id_focused_on_lp, - unsigned int lpn_len, char* lpn, cv::Rect& bbox); - bool TwoStageLPRPlatesTypeDetection(const int width,//width of image - const int height,//height of image i.e. the specified dimensions of the image - const int pixOpt,// pixel type : 1 (8 bpp greyscale image) 3 (RGB 24 bpp image) or 4 (RGBA 32 bpp image) - void* pbData, - unsigned int step,// source image bytes buffer - unsigned int id_global_view, - unsigned int id_focused_on_lp, - unsigned int id_plates_types_classifier, - unsigned int lpn_len, - char* lpn, - cv::Rect& bbox); - bool CloseDetector(unsigned int id); - bool ClosePlatesTypesClassifier(unsigned int id); - - private: - // Version-negotiated OrtApi pointer: caps at the DLL's max supported API level so - // that a newer SDK header (ORT_API_VERSION=22) paired with an older runtime DLL - // (e.g. ORT 1.17.1, API ≤17) does not emit "[ORT ERROR] API version not available". - // Mirrors the negotiation pattern used in EPLoader.cpp. - const OrtApi* g_ort = []() -> const OrtApi* { - const OrtApiBase* base = OrtGetApiBase(); - int dllMaxApi = ORT_API_VERSION; - const char* verStr = base->GetVersionString(); - int major = 0, minor = 0; - if (verStr && sscanf(verStr, "%d.%d", &major, &minor) == 2) - dllMaxApi = minor; - int targetApi = (ORT_API_VERSION < dllMaxApi) ? ORT_API_VERSION : dllMaxApi; - return base->GetApi(targetApi); - }(); - std::list detectors_envs; - std::list l_detectors_sessionOptions; - std::list detectors; - std::list detectors_ids; - std::list plates_types_envs; - std::list l_plates_types_classifier_sessionOptions; - std::list plates_types_classifiers; - std::list plates_types_classifier_ids; - }; -} -#endif \ No newline at end of file diff --git a/modules/ANSLPR/CMakeLists.txt b/modules/ANSLPR/CMakeLists.txt index 7f65c54..7a639b2 100644 --- a/modules/ANSLPR/CMakeLists.txt +++ b/modules/ANSLPR/CMakeLists.txt @@ -4,6 +4,7 @@ set(ANSLPR_SOURCES ANSLPR.cpp ANSLPR_CPU.cpp ANSLPR_OD.cpp + ANSLPR_OCR.cpp ANSGpuFrameRegistry.cpp GpuNV12SlotPool.cpp dllmain.cpp @@ -56,6 +57,7 @@ target_link_libraries(ANSLPR PRIVATE ANSODEngine PUBLIC ANSLicensingSystem # PUBLIC: Utility.h/SPDLogger symbols must be re-exported PRIVATE ANSMOT + PRIVATE ANSOCR # ANSALPR_OCR uses ANSONNXOCR from the OCR module PRIVATE labview PRIVATE spdlog_dep PRIVATE opencv diff --git a/modules/ANSLPR/dllmain.cpp b/modules/ANSLPR/dllmain.cpp index 2274aac..d0ba077 100644 --- a/modules/ANSLPR/dllmain.cpp +++ b/modules/ANSLPR/dllmain.cpp @@ -4,6 +4,7 @@ #include "ANSLPR.h" #include "ANSLPR_CPU.h" #include "ANSLPR_OD.h" +#include "ANSLPR_OCR.h" #include "ANSLibsLoader.h" #include "ANSGpuFrameRegistry.h" // gpu_frame_lookup(cv::Mat*) #include @@ -141,6 +142,9 @@ static int CreateANSALPRHandle_Impl(ANSCENTER::ANSALPR** Handle, const char* lic else if (engineType == 1) { (*Handle) = new ANSCENTER::ANSALPR_OD(); } + else if (engineType == 2) { + (*Handle) = new ANSCENTER::ANSALPR_OCR();// ONNX OCR (PaddleOCR v5) + } else { return 0; } @@ -757,6 +761,17 @@ extern "C" ANSLPR_API int ANSALPR_SetALPRCheckerEnabled(ANSCENTER::ANSALPR** } } +extern "C" ANSLPR_API int ANSALPR_SetCountry(ANSCENTER::ANSALPR** Handle, int country) { + if (!Handle || !*Handle) return -1; + try { + (*Handle)->SetCountry(static_cast(country)); + return 1; + } + catch (...) { + return 0; + } +} + extern "C" ANSLPR_API int ANSALPR_GetFormats(ANSCENTER::ANSALPR** Handle, LStrHandle Lstrformats)// semi separated formats { if (!Handle || !*Handle) return -1; diff --git a/modules/ANSOCR/ANSOCRBase.h b/modules/ANSOCR/ANSOCRBase.h index a625e91..185f7fe 100644 --- a/modules/ANSOCR/ANSOCRBase.h +++ b/modules/ANSOCR/ANSOCRBase.h @@ -181,7 +181,7 @@ namespace ANSCENTER { }; [[nodiscard]] virtual bool Destroy() = 0; }; - class ANSOCRUtility + class ANSOCR_API ANSOCRUtility { public: [[nodiscard]] static std::string OCRDetectionToJsonString(const std::vector& dets); diff --git a/modules/ANSOCR/dllmain.cpp b/modules/ANSOCR/dllmain.cpp index 83a7aeb..b4b922a 100644 --- a/modules/ANSOCR/dllmain.cpp +++ b/modules/ANSOCR/dllmain.cpp @@ -137,6 +137,18 @@ extern "C" ANSOCR_API int CreateANSOCRHandleEx(ANSCENTER::ANSOCRBase** Handle, // Ensure all shared DLLs (OpenCV, OpenVINO, TRT, ORT) are pre-loaded ANSCENTER::ANSLibsLoader::Initialize(); ANSCENTER::EngineType engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation(); + { + const char* vendorTag = + engineType == ANSCENTER::EngineType::NVIDIA_GPU ? "NVIDIA_GPU (TensorRT OCR enabled)" : + engineType == ANSCENTER::EngineType::AMD_GPU ? "AMD_GPU (ONNX Runtime / DirectML, TensorRT OCR DISABLED)" : + engineType == ANSCENTER::EngineType::OPENVINO_GPU ? "OPENVINO_GPU (ONNX Runtime / OpenVINO, TensorRT OCR DISABLED)" : + "CPU (ONNX Runtime, TensorRT OCR DISABLED)"; + char buf[192]; + snprintf(buf, sizeof(buf), + "[ANSOCR] CreateANSOCRHandleEx: detected engineType=%d [%s], engineMode=%d\n", + static_cast(engineType), vendorTag, engineMode); + OutputDebugStringA(buf); + } // Release existing handle if called twice (prevents leak from LabVIEW) if (*Handle) { @@ -159,13 +171,29 @@ extern "C" ANSOCR_API int CreateANSOCRHandleEx(ANSCENTER::ANSOCRBase** Handle, (*Handle) = new ANSCENTER::ANSCPUOCR(); } else { + // ANSRTOCR wraps PaddleOCRV5RTEngine which is strictly NVIDIA/CUDA: + // RTOCRDetector/Classifier/Recognizer use cv::cuda::GpuMat, cudaMalloc, + // cudaMemcpy and link against cudart. Instantiating it on AMD, Intel + // or pure-CPU machines either crashes in the CUDA runtime loader or + // hangs in amdkmdag when DirectML and TRT coexist. We therefore + // hard-gate the TRT path on NVIDIA_GPU and fall back to ANSONNXOCR + // (which uses CPU-side NV12 conversion and ONNX Runtime's EP auto- + // select, including DirectML for AMD). + const bool isNvidia = (engineType == ANSCENTER::EngineType::NVIDIA_GPU); switch (engineMode) { case 0:// Auto-detect, always use ONNX for better compatibility, especially on AMD GPUs and high-res images (*Handle) = new ANSCENTER::ANSONNXOCR(); break; - case 1:// GPU — use TensorRT engine. - limitSideLen = 960; - (*Handle) = new ANSCENTER::ANSRTOCR(); + case 1:// GPU — use TensorRT engine ONLY on NVIDIA hardware. + if (isNvidia) { + limitSideLen = 960; + (*Handle) = new ANSCENTER::ANSRTOCR(); + } else { + // AMD / Intel / CPU requested GPU mode — ANSRTOCR would crash. + // Fall back to ANSONNXOCR which picks the right ORT provider + // (DirectML on AMD, OpenVINO/CPU on Intel, CPU otherwise). + (*Handle) = new ANSCENTER::ANSONNXOCR(); + } break; case 2:// CPU (*Handle) = new ANSCENTER::ANSONNXOCR(); diff --git a/modules/ANSODEngine/ANSODVendorGate.h b/modules/ANSODEngine/ANSODVendorGate.h new file mode 100644 index 0000000..8ec7544 --- /dev/null +++ b/modules/ANSODEngine/ANSODVendorGate.h @@ -0,0 +1,57 @@ +#pragma once +// ANSODVendorGate.h — Cached NVIDIA hardware check for ANSODEngine.dll. +// +// ANSODEngine.dll links against CUDA::cudart_static + CUDA::cublas + +// CUDA::cublasLt and hosts the TensorRT inference classes (ANSRTYOLO, +// TENSORRTOD, TENSORRTCL, TENSORRTSEG, TENSORRTPOSE, ANSSAM3, ANSYOLOV10RTOD, +// ANSYOLOV12RTOD, ANSTENSORRTPOSE) plus the NV12 preprocess helper and the +// TRT engine pool. +// +// The dllmain factory already hard-gates TRT class instantiation on +// NVIDIA_GPU and falls back to ONNX Runtime / OpenVINO on AMD/Intel/CPU. +// However, several support paths still call into the CUDA runtime +// unconditionally: +// • GetNumGPUs() / GetPoolMaxSlotsPerGpu() / CheckGPUVRAM() helpers +// (called from inside NVIDIA_GPU guards today, but safer to gate at +// source so a future refactor cannot accidentally wake up cudart on +// AMD/Intel). +// • A few case labels in the model-type switch instantiate TRT classes +// without an explicit NVIDIA_GPU check — they are currently unreachable +// due to upstream modelType rewriting, but leaving them unguarded +// creates a maintenance trap. +// +// Solution: a single process-wide cached predicate that evaluates +// CheckHardwareInformation() exactly once. On AMD/Intel/CPU the predicate +// returns false and every gated site short-circuits before touching any +// CUDA API. +// +// Mirrors ANSCVVendorGate / ANSLPR_OD::isNvidiaEngine / ANSOCR factory gate +// / ANSFR CreateANSRFHandle vendor log. Keeps the four shipped DLLs on a +// single, auditable pattern. + +#include "ANSLicense.h" +#include + +namespace ansod_vendor_gate { + +// Lazily evaluates ANSLicenseHelper::CheckHardwareInformation() once and +// caches the result. Thread-safe via std::atomic (0 = unknown, +// 1 = NVIDIA, 2 = non-NVIDIA). No std::call_once overhead on the hot +// inference path. Fails safe to non-NVIDIA on exception. +[[nodiscard]] inline bool IsNvidiaGpuAvailable() noexcept { + static std::atomic s_state{0}; + int cached = s_state.load(std::memory_order_acquire); + if (cached != 0) return cached == 1; + try { + const ANSCENTER::EngineType detected = + ANSCENTER::ANSLicenseHelper::CheckHardwareInformation(); + const bool isNvidia = (detected == ANSCENTER::EngineType::NVIDIA_GPU); + s_state.store(isNvidia ? 1 : 2, std::memory_order_release); + return isNvidia; + } catch (...) { + s_state.store(2, std::memory_order_release); + return false; + } +} + +} // namespace ansod_vendor_gate diff --git a/modules/ANSODEngine/ANSONNXYOLO.cpp b/modules/ANSODEngine/ANSONNXYOLO.cpp index caef52f..b50d4df 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.cpp +++ b/modules/ANSODEngine/ANSONNXYOLO.cpp @@ -237,6 +237,38 @@ namespace ANSCENTER { output_node_names.data(), num_outputs); + // ── Output shape sanity check ─────────────────────────────────── + // DirectML on some AMD configurations has been observed to return + // output tensors whose dim[1]/dim[2] values don't match what the + // ONNX graph actually produced, which propagates into + // postprocessLegacy / postprocessEndToEnd as huge numBoxes / + // numChannels values and causes multi-terabyte cv::Mat allocations + // inside the `cv::Mat(numChannels, numBoxes, CV_32F, ...).t()` + // call (observed as "Failed to allocate 3522082959360 bytes" on + // Ryzen APUs). Bail out early here instead of letting the + // postprocess layer try to materialise a 3.5 TB buffer. + // + // Sane upper bounds for Ultralytics YOLO outputs: + // • legacy [1, 84..300, 8400..25200] → max dim ≈ 30k + // • end2end [1, 300, 6..56] → max dim ≈ 300 + // • segmentation proto mask [1, 32, 160, 160] → max dim ≈ 160 + // • classification [1, 1000] → max dim ≈ 1k + // 1,000,000 is ~30x the largest real-world dim and catches the + // garbage values without clipping any legitimate model. + constexpr int64_t kMaxOutputDim = 1000000; + for (size_t t = 0; t < outputTensors.size(); ++t) { + const auto shape = outputTensors[t].GetTensorTypeAndShapeInfo().GetShape(); + for (size_t d = 0; d < shape.size(); ++d) { + if (shape[d] < 0 || shape[d] > kMaxOutputDim) { + std::cerr << "[ONNXYOLO] detect: output[" << t + << "] dim[" << d << "]=" << shape[d] + << " is out of range — refusing to postprocess." + << std::endl; + return {}; + } + } + } + const cv::Size resizedShape( static_cast(input_node_dims[3]), static_cast(input_node_dims[2])); @@ -1399,6 +1431,23 @@ namespace ANSCENTER { output_node_names.data(), num_outputs); + // Output shape sanity check — see detect() for rationale. Prevents + // DirectML-returned garbage dims from propagating into postprocess + // and triggering multi-terabyte cv::Mat allocations on AMD. + constexpr int64_t kMaxOutputDim = 1000000; + for (size_t t = 0; t < outputTensors.size(); ++t) { + const auto sh = outputTensors[t].GetTensorTypeAndShapeInfo().GetShape(); + for (size_t d = 0; d < sh.size(); ++d) { + if (sh[d] < 0 || sh[d] > kMaxOutputDim) { + std::cerr << "[ONNXYOLO] detectBatch: output[" << t + << "] dim[" << d << "]=" << sh[d] + << " is out of range — refusing to postprocess." + << std::endl; + return std::vector>(N); + } + } + } + const cv::Size resizedShape( static_cast(input_node_dims[3]), static_cast(input_node_dims[2])); @@ -1589,59 +1638,92 @@ namespace ANSCENTER { } // ======================================================================== - // WarmUpEngine — run 2 dummy inferences after session creation + // WarmUpEngine — run a dummy inference after session creation. // - // On AMD RDNA2 iGPUs (e.g. Radeon 680M on Ryzen 6000-series APUs), the - // very first detect() call triggers DirectML shader compile + GPU kernel - // cache population for the entire YOLO graph. That first pass can - // legitimately take several seconds of sustained GPU work, which is long - // enough to coincide with TDR watchdog firing and has triggered - // amdkmdag.sys bugchecks at +0xf03d under DirectML 1.15.4 (the latest). + // Scope: **NVIDIA (CUDA EP) only.** On first inference, the CUDA EP + // allocates its memory arena (capped at 2 GB via BasicOrtHandler config), + // resolves cuDNN convolution algorithms, and populates the kernel launch + // cache. Running one dummy inference at load time amortises this cost + // so the first real frame doesn't see a latency spike. // - // Running 2 dummy inferences at startup burns the compile cost under - // controlled conditions so that the first real frame is already fast. - // The second call should always be quick and confirms the cache is warm. + // Explicitly disabled on AMD, Intel and CPU: + // • AMD (DirectML) — calling detect() at load time has been observed + // to hit a multi-terabyte cv::Mat allocation inside postprocessLegacy + // on AMD RDNA iGPUs when DirectML returns garbage output tensor + // dims. ONNXYOLO::detect() now has an output-shape sanity guard + // that catches this at runtime, so the warm-up would add risk + // without benefit. Earlier builds enabled warm-up specifically for + // Radeon 680M TDR mitigation; that workaround is obsolete with + // current DirectML 1.15.x drivers. + // • Intel (OpenVINO) — running detect() at load time has been + // observed to expose latent heap-corruption bugs + // (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION 0xc0000374). + // • CPU EP — no shader compile or kernel cache to warm up; the first + // real frame has the same latency as any subsequent frame. // - // Non-fatal on failure: if warm-up itself crashes, regular inference may - // still succeed, or will fail with a clearer error message. + // Non-fatal on failure: if warm-up itself throws, regular inference + // still works — the engine is fully loaded before WarmUpEngine runs. // ======================================================================== void ANSONNXYOLO::WarmUpEngine() { if (!m_ortEngine) return; - // Warm-up exists solely to pre-compile DirectML shaders on AMD RDNA2 - // iGPUs (Radeon 680M). It has no benefit on CPU / OpenVINO / CUDA - // and running detect() at load time has been observed to expose - // latent heap-corruption bugs (ntdll +0x1176e5 / STATUS_HEAP_CORRUPTION - // 0xc0000374) on Intel machines. Gate strictly on AMD_GPU. - if (m_ortEngine->getEngineType() != EngineType::AMD_GPU) { - ANS_DBG("ONNXYOLO", "Warm-up skipped (non-AMD EP)"); + // Gate strictly on NVIDIA_GPU. Every other EP is a no-op. + if (m_ortEngine->getEngineType() != EngineType::NVIDIA_GPU) { + ANS_DBG("ONNXYOLO", "Warm-up skipped (non-NVIDIA EP)"); return; } - try { - const int w = _modelConfig.inpWidth > 0 ? _modelConfig.inpWidth : 640; - const int h = _modelConfig.inpHeight > 0 ? _modelConfig.inpHeight : 640; + // ── Strict dimension validation ───────────────────────────────── + // Defensive: refuse to warm up with implausible model dimensions. + // _modelConfig values come from the caller's ModelConfig and are + // normally 224..640; anything outside [32, 4096] is almost certainly + // a bug in the caller and we skip warm-up rather than risk a huge + // cv::Mat allocation inside detect(). + constexpr int kMinDim = 32; + constexpr int kMaxDim = 4096; + const int rawW = _modelConfig.inpWidth; + const int rawH = _modelConfig.inpHeight; + if (rawW <= 0 || rawH <= 0 || rawW > kMaxDim || rawH > kMaxDim) { + _logger.LogWarn("ANSONNXYOLO::WarmUpEngine", + "Warm-up skipped — suspect input dims (" + + std::to_string(rawW) + "x" + std::to_string(rawH) + ")", + __FILE__, __LINE__); + return; + } + const int w = std::clamp(rawW, kMinDim, kMaxDim); + const int h = std::clamp(rawH, kMinDim, kMaxDim); + try { // Mid-gray BGR image matches the letterbox fill colour used in // preprocessing (114,114,114 ~ 128) and avoids degenerate inputs. cv::Mat dummy(h, w, CV_8UC3, cv::Scalar(128, 128, 128)); - ANS_DBG("ONNXYOLO", "Warm-up: running 2 dummy inferences (%dx%d)", w, h); + ANS_DBG("ONNXYOLO", "Warm-up: running 1 dummy CUDA inference (%dx%d)", w, h); - for (int i = 0; i < 2; ++i) { - auto t0 = std::chrono::steady_clock::now(); - (void)m_ortEngine->detect(dummy, _classes, - PROBABILITY_THRESHOLD, - NMS_THRESHOLD, - NUM_KPS); - auto t1 = std::chrono::steady_clock::now(); - auto ms = std::chrono::duration_cast(t1 - t0).count(); - ANS_DBG("ONNXYOLO", "Warm-up #%d: %lld ms", i, (long long)ms); - } + auto t0 = std::chrono::steady_clock::now(); + (void)m_ortEngine->detect(dummy, _classes, + PROBABILITY_THRESHOLD, + NMS_THRESHOLD, + NUM_KPS); + auto t1 = std::chrono::steady_clock::now(); + auto ms = std::chrono::duration_cast(t1 - t0).count(); + ANS_DBG("ONNXYOLO", "Warm-up done: %lld ms", (long long)ms); + } + catch (const cv::Exception& e) { + // Defensive — should not fire on NVIDIA CUDA EP, but if it does + // the engine itself is still loaded and real inference will work. + _logger.LogWarn("ANSONNXYOLO::WarmUpEngine", + std::string("Warm-up skipped (cv::Exception, non-fatal): ") + e.what(), + __FILE__, __LINE__); } catch (const std::exception& e) { - _logger.LogError("ANSONNXYOLO::WarmUpEngine", - std::string("Warm-up failed (non-fatal): ") + e.what(), + _logger.LogWarn("ANSONNXYOLO::WarmUpEngine", + std::string("Warm-up skipped (std::exception, non-fatal): ") + e.what(), + __FILE__, __LINE__); + } + catch (...) { + _logger.LogWarn("ANSONNXYOLO::WarmUpEngine", + "Warm-up skipped (unknown exception, non-fatal)", __FILE__, __LINE__); } } diff --git a/modules/ANSODEngine/dllmain.cpp b/modules/ANSODEngine/dllmain.cpp index aa310c0..651a145 100644 --- a/modules/ANSODEngine/dllmain.cpp +++ b/modules/ANSODEngine/dllmain.cpp @@ -7,6 +7,7 @@ #include "engine/EnginePoolManager.h" // clearAll() on DLL_PROCESS_DETACH #include // INT_MIN #include "ANSLicense.h" // ANS_DBG macro for DebugView +#include "ANSODVendorGate.h" // ansod_vendor_gate::IsNvidiaGpuAvailable() // Process-wide flag: when true, all engines force single-GPU path (no pool, no idle timers). // Defined here, declared extern in EngineBuildLoadNetwork.inl. @@ -88,6 +89,17 @@ static std::mutex g_gpuCountMutex; static int GetNumGPUs() { std::lock_guard lk(g_gpuCountMutex); if (g_numGPUs < 0) { + // Defense-in-depth: all callers (AssignNextGPU, GetPoolMaxSlotsPerGpu, + // CheckGPUVRAM) are invoked inside factory-level NVIDIA_GPU guards, + // but skip the CUDA runtime entirely on AMD/Intel/CPU hardware so a + // future refactor cannot accidentally wake up cudart on non-NVIDIA. + // See ANSODVendorGate.h. + if (!ansod_vendor_gate::IsNvidiaGpuAvailable()) { + g_numGPUs = 1; // report a single "virtual" slot so round-robin is a no-op + std::cout << "Info [GPU]: non-NVIDIA hardware — CUDA probe skipped, pool slots=1" + << std::endl; + return g_numGPUs; + } // Use yield mode before any CUDA call to avoid busy-wait spinning // that falsely reports 100% GPU utilization in nvidia-smi. cudaSetDeviceFlags(cudaDeviceScheduleYield); @@ -108,6 +120,13 @@ static int GetPoolMaxSlotsPerGpu() { static std::mutex s_mutex; std::lock_guard lk(s_mutex); if (s_result != INT_MIN) return s_result; + // Short-circuit on non-NVIDIA: no TRT engines will be built, no pool to + // size, and cudaSetDevice/cudaMemGetInfo below should not be reached. + // Safety net — callers today are already inside NVIDIA_GPU guards. + if (!ansod_vendor_gate::IsNvidiaGpuAvailable()) { + s_result = 1; + return s_result; + } const int n = GetNumGPUs(); if (n <= 1) { s_result = 1; @@ -132,6 +151,9 @@ static int GetPoolMaxSlotsPerGpu() { // Returns the next GPU index in round-robin order. // Thread-safe: uses atomic fetch_add. static int AssignNextGPU() { + // Non-NVIDIA short-circuit: no CUDA devices, return 0 and skip the + // "assigning task" log to avoid polluting AMD/Intel/CPU logs. + if (!ansod_vendor_gate::IsNvidiaGpuAvailable()) return 0; const int numGPUs = GetNumGPUs(); const int idx = g_gpuRoundRobinCounter.fetch_add(1); const int gpuIndex = idx % numGPUs; @@ -144,6 +166,11 @@ static int AssignNextGPU() { // Returns true if sufficient, false if not. // minFreeBytes: minimum free VRAM required (default 512 MiB safety margin). static bool CheckGPUVRAM(int gpuIndex, size_t minFreeBytes = 512ULL * 1024 * 1024) { + // Non-NVIDIA short-circuit: no CUDA devices present — report "OK" + // silently so the TRT pool path is a no-op on AMD/Intel/CPU and the + // log isn't polluted with spurious 0-byte VRAM warnings. + if (!ansod_vendor_gate::IsNvidiaGpuAvailable()) return true; + int prevDevice = 0; cudaGetDevice(&prevDevice); cudaSetDevice(gpuIndex); @@ -253,6 +280,16 @@ BOOL APIENTRY DllMain( HMODULE hModule, // Pin the DLL so it is never unmapped while idle-timer or CUDA threads // are still running. During LabVIEW shutdown the CLR/COM teardown can // unload DLLs before all threads exit → crash at unmapped code. + // + // CRITICAL: do NOT call CheckHardwareInformation() or + // ansod_vendor_gate::IsNvidiaGpuAvailable() from here. DllMain holds + // the OS loader lock (LdrpLoaderLock). CheckHardwareInformation + // touches hwinfo → DXGI / WMI / COM, which internally call + // LoadLibrary; doing that while holding the loader lock causes a + // classic loader-lock deadlock (observed as a full hang of the + // ANSLPR-UnitTest stress test). The vendor gate will lazy- + // initialise on the first real call from worker code, which runs + // with the loader lock released. { HMODULE hSelf = nullptr; GetModuleHandleExW( @@ -511,8 +548,19 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase** modelConfig.modelType = ANSCENTER::ModelType::ODHUBMODEL; break; case 14: //TensorRT for Object Detection Yolov10 - (*Handle) = new ANSCENTER::ANSYOLOV10RTOD(); - modelConfig.modelType = ANSCENTER::ModelType::YOLOV10RTOD; + // Upstream modelType rewrite (see top of each factory) already + // redirects 14 → 31 (RTYOLO) on NVIDIA or 14 → 30 (ONNXYOLO) on + // non-NVIDIA, so this branch is unreachable in practice. Keep + // an explicit vendor gate as defense-in-depth against future + // refactors — ANSYOLOV10RTOD is a TensorRT class and must never + // be constructed on AMD/Intel/CPU hardware. + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + (*Handle) = new ANSCENTER::ANSYOLOV10RTOD(); + modelConfig.modelType = ANSCENTER::ModelType::YOLOV10RTOD; + } else { + (*Handle) = new ANSCENTER::ANSONNXYOLO(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO; + } break; case 15: //OpenVino for Object Detection Yolov10 (*Handle) = new ANSCENTER::ANSOYOLOV10OVOD(); @@ -832,8 +880,19 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl modelConfig.modelType = ANSCENTER::ModelType::ODHUBMODEL; break; case 14: //TensorRT for Object Detection Yolov10 - (*Handle) = new ANSCENTER::ANSYOLOV10RTOD(); - modelConfig.modelType = ANSCENTER::ModelType::YOLOV10RTOD; + // Upstream modelType rewrite (see top of each factory) already + // redirects 14 → 31 (RTYOLO) on NVIDIA or 14 → 30 (ONNXYOLO) on + // non-NVIDIA, so this branch is unreachable in practice. Keep + // an explicit vendor gate as defense-in-depth against future + // refactors — ANSYOLOV10RTOD is a TensorRT class and must never + // be constructed on AMD/Intel/CPU hardware. + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + (*Handle) = new ANSCENTER::ANSYOLOV10RTOD(); + modelConfig.modelType = ANSCENTER::ModelType::YOLOV10RTOD; + } else { + (*Handle) = new ANSCENTER::ANSONNXYOLO(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO; + } break; case 15: //OpenVino for Object Detection Yolov10 (*Handle) = new ANSCENTER::ANSOYOLOV10OVOD(); @@ -1193,8 +1252,19 @@ extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase** modelConfig.modelType = ANSCENTER::ModelType::ODHUBMODEL; break; case 14: //TensorRT for Object Detection Yolov10 - (*Handle) = new ANSCENTER::ANSYOLOV10RTOD(); - modelConfig.modelType = ANSCENTER::ModelType::YOLOV10RTOD; + // Upstream modelType rewrite (see top of each factory) already + // redirects 14 → 31 (RTYOLO) on NVIDIA or 14 → 30 (ONNXYOLO) on + // non-NVIDIA, so this branch is unreachable in practice. Keep + // an explicit vendor gate as defense-in-depth against future + // refactors — ANSYOLOV10RTOD is a TensorRT class and must never + // be constructed on AMD/Intel/CPU hardware. + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + (*Handle) = new ANSCENTER::ANSYOLOV10RTOD(); + modelConfig.modelType = ANSCENTER::ModelType::YOLOV10RTOD; + } else { + (*Handle) = new ANSCENTER::ANSONNXYOLO(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO; + } break; case 15: //OpenVino for Object Detection Yolov10 (*Handle) = new ANSCENTER::ANSOYOLOV10OVOD(); diff --git a/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp b/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp index b56927b..f51deb8 100644 --- a/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp +++ b/tests/ANSLPR-UnitTest/ANSLPR-UnitTest.cpp @@ -1,4 +1,5 @@ -#include +#define NOMINMAX +#include #include #include #include @@ -31,6 +32,60 @@ #include #include "EPLoader.h" +// Decode \\uXXXX (literal backslash-u-hex) sequences back to UTF-8. +// VectorDetectionToJsonString double-escapes Unicode for LabVIEW compatibility, +// so JSON strings contain literal "\u54c1" text instead of actual Unicode chars. +static std::string DecodeUnicodeEscapes(const std::string& input) { + std::string result; + result.reserve(input.size()); + size_t i = 0; + while (i < input.size()) { + if (i + 5 < input.size() && input[i] == '\\' && input[i + 1] == 'u') { + // Parse 4 hex digits + std::string hex = input.substr(i + 2, 4); + char* end = nullptr; + uint32_t cp = static_cast(strtoul(hex.c_str(), &end, 16)); + if (end == hex.c_str() + 4) { + // Check for surrogate pair (\\uD800-DBFF followed by \\uDC00-DFFF) + if (cp >= 0xD800 && cp <= 0xDBFF && i + 11 < input.size() + && input[i + 6] == '\\' && input[i + 7] == 'u') { + std::string hex2 = input.substr(i + 8, 4); + uint32_t cp2 = static_cast(strtoul(hex2.c_str(), &end, 16)); + if (end == hex2.c_str() + 4 && cp2 >= 0xDC00 && cp2 <= 0xDFFF) { + cp = 0x10000 + ((cp - 0xD800) << 10) + (cp2 - 0xDC00); + i += 12; + } else { + i += 6; + } + } else { + i += 6; + } + // Encode codepoint as UTF-8 + if (cp < 0x80) { + result += static_cast(cp); + } else if (cp < 0x800) { + result += static_cast(0xC0 | (cp >> 6)); + result += static_cast(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + result += static_cast(0xE0 | (cp >> 12)); + result += static_cast(0x80 | ((cp >> 6) & 0x3F)); + result += static_cast(0x80 | (cp & 0x3F)); + } else { + result += static_cast(0xF0 | (cp >> 18)); + result += static_cast(0x80 | ((cp >> 12) & 0x3F)); + result += static_cast(0x80 | ((cp >> 6) & 0x3F)); + result += static_cast(0x80 | (cp & 0x3F)); + } + } else { + result += input[i++]; + } + } else { + result += input[i++]; + } + } + return result; +} + template T GetOptionalValue(const boost::property_tree::ptree& pt, std::string attribute, T defaultValue) { if (pt.count(attribute)) { @@ -3533,8 +3588,229 @@ int ANSLPR_OD_CPU_VideoTest() { return (frameIndex > 0) ? 0 : -4; } +// ── ANSALPR_OCR test: Japanese license plate detection using ANSONNXOCR ── +// Render UTF-8 text onto a cv::Mat using Windows GDI (supports CJK/Unicode). +// cv::putText only handles ASCII — Japanese characters render as '?'. +#ifdef WIN32 +static void putTextUnicode(cv::Mat& img, const std::string& text, cv::Point org, + double fontScale, cv::Scalar color, int thickness) { + int wlen = MultiByteToWideChar(CP_UTF8, 0, text.c_str(), -1, nullptr, 0); + std::wstring wtext(wlen - 1, 0); + MultiByteToWideChar(CP_UTF8, 0, text.c_str(), -1, &wtext[0], wlen); + + HDC hdc = CreateCompatibleDC(nullptr); + int fontHeight = (int)(fontScale * 30); + + HFONT hFont = CreateFontW(fontHeight, 0, 0, 0, + (thickness > 2) ? FW_BOLD : FW_NORMAL, + FALSE, FALSE, FALSE, + DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, CLIP_DEFAULT_PRECIS, + ANTIALIASED_QUALITY, DEFAULT_PITCH | FF_SWISS, L"Yu Gothic UI"); + HFONT hOldFont = (HFONT)SelectObject(hdc, hFont); + + SIZE sz; + GetTextExtentPoint32W(hdc, wtext.c_str(), (int)wtext.size(), &sz); + + BITMAPINFO bmi = {}; + bmi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER); + bmi.bmiHeader.biWidth = sz.cx; + bmi.bmiHeader.biHeight = -sz.cy; + bmi.bmiHeader.biPlanes = 1; + bmi.bmiHeader.biBitCount = 32; + bmi.bmiHeader.biCompression = BI_RGB; + void* bits = nullptr; + HBITMAP hBmp = CreateDIBSection(hdc, &bmi, DIB_RGB_COLORS, &bits, nullptr, 0); + HBITMAP hOldBmp = (HBITMAP)SelectObject(hdc, hBmp); + + SetBkMode(hdc, TRANSPARENT); + SetTextColor(hdc, RGB((int)color[2], (int)color[1], (int)color[0])); + TextOutW(hdc, 0, 0, wtext.c_str(), (int)wtext.size()); + + cv::Mat textImg(sz.cy, sz.cx, CV_8UC4, bits); + for (int row = 0; row < sz.cy; ++row) { + for (int col = 0; col < sz.cx; ++col) { + cv::Vec4b px = textImg.at(row, col); + if (px[0] != 0 || px[1] != 0 || px[2] != 0) { + int dy = org.y + row; + int dx = org.x + col; + if (dy >= 0 && dy < img.rows && dx >= 0 && dx < img.cols) { + img.at(dy, dx) = cv::Vec3b(px[0], px[1], px[2]); + } + } + } + } + + SelectObject(hdc, hOldBmp); + SelectObject(hdc, hOldFont); + DeleteObject(hBmp); + DeleteObject(hFont); + DeleteDC(hdc); +} +#endif + +int ALPR_OCR_Test() { + std::cout << "=== ALPR_OCR_Test: Japanese License Plate (ANSALPR_OCR) ===" << std::endl; + std::filesystem::path currentPath = std::filesystem::current_path(); + std::cout << "Current working directory: " << currentPath << std::endl; + + ANSCENTER::ANSALPR* infHandle = nullptr; + std::string licenseKey = ""; + std::string modelFilePath = "C:\\Projects\\ANSVIS\\Models\\ANS_GenericALPR_v2.0.zip"; + std::string imagePath = "C:\\Programs\\ModelTraining\\JLPD\\data\\test7.jpg"; + + int engineType = 2; // ANSALPR_OCR + double detectionThreshold = 0.3; + double ocrThreshold = 0.5; + double colourThreshold = 0.0; // No colour detection for this test + + // Step 1: Create handle + int createResult = CreateANSALPRHandle(&infHandle, licenseKey.c_str(), + modelFilePath.c_str(), "", engineType, detectionThreshold, ocrThreshold, colourThreshold); + std::cout << "CreateANSALPRHandle result: " << createResult << std::endl; + if (!createResult || !infHandle) { + std::cerr << "Failed to create ANSALPR_OCR handle" << std::endl; + return -1; + } + + // Step 2: Set country to Japan + ANSALPR_SetCountry(&infHandle, 5); // JAPAN = 5 + std::cout << "Country set to JAPAN" << std::endl; + + // Step 3: Load engine + auto engineStart = std::chrono::high_resolution_clock::now(); + int loadResult = LoadANSALPREngineHandle(&infHandle); + auto engineEnd = std::chrono::high_resolution_clock::now(); + double engineMs = std::chrono::duration(engineEnd - engineStart).count(); + std::cout << "LoadANSALPREngineHandle result: " << loadResult << " (" << engineMs << " ms)" << std::endl; + if (!loadResult) { + std::cerr << "Failed to load ANSALPR_OCR engine" << std::endl; + ReleaseANSALPRHandle(&infHandle); + return -2; + } + + // Step 4: Load image + cv::Mat input = cv::imread(imagePath, cv::IMREAD_COLOR); + if (input.empty()) { + std::cerr << "Failed to load image: " << imagePath << std::endl; + ReleaseANSALPRHandle(&infHandle); + return -3; + } + std::cout << "Image loaded: " << input.cols << "x" << input.rows << std::endl; + + cv::Mat frame = input.clone(); + int width = frame.cols; + int height = frame.rows; + + // Convert to raw BGR bytes for RunInferenceBinary + unsigned int bufferLength = static_cast(frame.total() * frame.elemSize()); + unsigned char* imageBytes = new unsigned char[bufferLength]; + std::memcpy(imageBytes, frame.data, bufferLength); + + // Step 5: Warmup run + auto warmupStart = std::chrono::high_resolution_clock::now(); + std::string detectionResult = ANSALPR_RunInferenceBinary(&infHandle, imageBytes, width, height); + auto warmupEnd = std::chrono::high_resolution_clock::now(); + double warmupMs = std::chrono::duration(warmupEnd - warmupStart).count(); + std::cout << "Warmup inference: " << warmupMs << " ms" << std::endl; + std::cout << "ALPR Result: " << detectionResult << std::endl; + + // Step 6: Benchmark + const int benchmarkIterations = 10; + std::vector times; + times.reserve(benchmarkIterations); + for (int i = 0; i < benchmarkIterations; ++i) { + auto t0 = std::chrono::high_resolution_clock::now(); + std::string result = ANSALPR_RunInferenceBinary(&infHandle, imageBytes, width, height); + auto t1 = std::chrono::high_resolution_clock::now(); + double ms = std::chrono::duration(t1 - t0).count(); + times.push_back(ms); + std::cout << " Run " << (i + 1) << "/" << benchmarkIterations << ": " << ms << " ms" << std::endl; + } + std::sort(times.begin(), times.end()); + double sum = std::accumulate(times.begin(), times.end(), 0.0); + double avg = sum / benchmarkIterations; + double median = (benchmarkIterations % 2 == 0) + ? (times[benchmarkIterations / 2 - 1] + times[benchmarkIterations / 2]) / 2.0 + : times[benchmarkIterations / 2]; + std::cout << "\n=== Benchmark (" << benchmarkIterations << " runs) ===" << std::endl; + std::cout << " Avg: " << avg << " ms" << std::endl; + std::cout << " Median: " << median << " ms" << std::endl; + std::cout << " Min: " << times.front() << " ms" << std::endl; + std::cout << " Max: " << times.back() << " ms" << std::endl; + std::cout << " FPS: " << (1000.0 / avg) << std::endl; + + delete[] imageBytes; + + // Step 7: Draw results on image + if (!detectionResult.empty()) { + try { + boost::property_tree::ptree pt; + std::stringstream ss(detectionResult); + boost::property_tree::read_json(ss, pt); + BOOST_FOREACH(const boost::property_tree::ptree::value_type& child, pt.get_child("results")) { + const boost::property_tree::ptree& res = child.second; + const auto class_name_raw = GetData(res, "class_name"); + const std::string class_name = DecodeUnicodeEscapes(class_name_raw); + const auto x = GetData(res, "x"); + const auto y = GetData(res, "y"); + const auto w = GetData(res, "width"); + const auto h = GetData(res, "height"); + + cv::rectangle(frame, cv::Rect(x, y, w, h), cv::Scalar(0, 255, 0), 2); + + std::string extraInfo = GetOptionalValue(res, "extra_info", ""); + std::cout << " Plate: " << class_name << std::endl; + if (!extraInfo.empty()) { + std::cout << " extra_info: " << extraInfo << std::endl; + } + +#ifdef WIN32 + { + int textH = (int)(1.5 * 30); + int ty = y - 5 - textH; + if (ty < 0) ty = y + 3; + putTextUnicode(frame, class_name, cv::Point(x, ty), + 1.5, cv::Scalar(0, 0, 255), 3); + } +#else + cv::putText(frame, class_name, cv::Point(x, y - 5), + cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 0, 255), 2, cv::LINE_AA); +#endif + } + } + catch (const std::exception& e) { + std::cerr << "JSON parse error: " << e.what() << std::endl; + } + } + + // Step 8: Display result + cv::Mat display; + double scale = std::min(1920.0 / frame.cols, 1080.0 / frame.rows); + if (scale < 1.0) { + cv::resize(frame, display, cv::Size(), scale, scale); + } else { + display = frame; + } + cv::namedWindow("ALPR_OCR_Test", cv::WINDOW_AUTOSIZE); + cv::imshow("ALPR_OCR_Test", display); + cv::waitKey(0); + + // Cleanup + ReleaseANSALPRHandle(&infHandle); + cv::destroyAllWindows(); + frame.release(); + input.release(); + + std::cout << "=== ALPR_OCR_Test complete ===" << std::endl; + return 0; +} + int main() { +#ifdef WIN32 + SetConsoleOutputCP(CP_UTF8); + SetConsoleCP(CP_UTF8); +#endif // ANSLPR_OD_INDOInferences_FileTest(); //ANSLPR_OD_Inferences_FileTest(); //ANSLPR_OD_VideoTest(); @@ -3544,11 +3820,12 @@ int main() // ANSLPR_CPU_Inferences_FileTest(); //} //ANSLPR_SingleTask_Test(); - ANSLPR_CPU_StressTest(); + //ANSLPR_CPU_StressTest(); //ANSLPR_MultiGPU_StressTest(); //ANSLPR_MultiGPU_StressTest_SimulatedCam(); // ANSLPR_MultiGPU_StressTest_FilePlayer(); //ANSLPR_OD_CPU_VideoTest(); + ALPR_OCR_Test(); return 0; }