diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 4344da6..527ad7d 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -211,7 +211,16 @@ "Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\)\")", "Bash(python -c \"import reportlab.pdfbase.pdfmetrics; print\\('pages: \\(pypdf not available — file size confirms\\)'\\)\")", "Bash(python -m pip install pypdf --quiet)", - "Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\); print\\('first page text \\(head\\):', r.pages[0].extract_text\\(\\)[:300].replace\\(chr\\(10\\), ' '\\)\\)\")" + "Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\); print\\('first page text \\(head\\):', r.pages[0].extract_text\\(\\)[:300].replace\\(chr\\(10\\), ' '\\)\\)\")", + "Bash(awk -F'\\\\t' '{print NR}' \"C:/Users/nghia/Downloads/ANSLEGION.log1.log\")", + "Bash(awk -F'\\\\t' 'END {print \"lines:\", NR, \" duration_s:\", $2, \" rate_lines/s:\", NR/$2}' \"C:/Users/nghia/Downloads/ANSLEGION.log1.log\")", + "Bash(grep -nE \"RunInference|RunInferencesBatch|infer\\\\\\\\\\(\\\\\\\\\\)|set_input_tensor|inference_request|m_inferRequest|mutex|lock_guard\" C:/Projects/CLionProjects/ANSCORE/engines/OpenVINOEngine/src/**/*.cpp C:/Projects/CLionProjects/ANSCORE/engines/OpenVINOEngine/include/**/*.h)", + "Bash(grep -nE \"DetectObjects|TryLockWithTimeout|ortRun|->Run\\\\\\\\\\(|m_ortEngine|infer|RunInference\" C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.cpp)", + "Bash(grep -nE \"ONNXYOLO::detect|ort_session->Run|set_input|detect\\\\\\\\\\(.*const.*Mat|m_inputBuffer|m_input_tensor|allocator|ortInput|run_options|preprocess|postprocess|class ONNXYOLO\" C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.cpp C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.h)", + "Bash(ninja ANSODEngine)", + "PowerShell($env:Path = \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Tools\\\\Llvm\\\\x64\\\\bin;\" + $env:Path; & \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvarsall.bat\" x64 2>&1 | Out-Null; Get-Command ninja -ErrorAction SilentlyContinue | Select-Object Source)", + "PowerShell($ninjaPaths = @\\(\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\Common7\\\\IDE\\\\CommonExtensions\\\\Microsoft\\\\CMake\\\\Ninja\\\\ninja.exe\", \"C:\\\\ProgramData\\\\chocolatey\\\\bin\\\\ninja.exe\", \"C:\\\\tools\\\\ninja\\\\ninja.exe\"\\); foreach \\($p in $ninjaPaths\\) { if \\(Test-Path $p\\) { Write-Output $p } })", + "PowerShell(cmd /c '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvarsall.bat\" x64 >nul 2>&1 && cd /d C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release && \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\Common7\\\\IDE\\\\CommonExtensions\\\\Microsoft\\\\CMake\\\\Ninja\\\\ninja.exe\" ANSODEngine 2>&1')" ] } } diff --git a/modules/ANSODEngine/ANSONNXYOLO.cpp b/modules/ANSODEngine/ANSONNXYOLO.cpp index 77adb64..2f17bef 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.cpp +++ b/modules/ANSODEngine/ANSONNXYOLO.cpp @@ -82,9 +82,15 @@ namespace ANSCENTER { } // ------------------------------------------------------------------ - // transform — BGR → RGB, letterbox, /255, HWC→CHW + // transformInto — BGR → RGB, letterbox, /255, HWC→CHW into a + // caller-provided buffer + dims vector. Thread-safe across instances: + // each call's tensor backing memory is unique to the caller, so two + // threads sharing one ONNXYOLO can preprocess in parallel without + // racing on a shared input_values_handler. // ------------------------------------------------------------------ - Ort::Value ONNXYOLO::transform(const cv::Mat& mat) + Ort::Value ONNXYOLO::transformInto(const cv::Mat& mat, + std::vector& outBuffer, + std::vector& outDims) { // Grayscale → BGR if needed cv::Mat bgrMat; @@ -94,7 +100,10 @@ namespace ANSCENTER { bgrMat = mat; } - // Check if model is classification (first output has 2 dims: [B, nc]) + // Check if model is classification (first output has 2 dims: [B, nc]). + // output_node_dims is populated once at session creation in + // BasicOrtHandler::initialize_handler and never mutated by this + // class, so reading it concurrently from multiple threads is safe. const bool isClassification = !output_node_dims.empty() && output_node_dims[0].size() == 2; @@ -116,7 +125,7 @@ namespace ANSCENTER { const int width = canvas.cols; const size_t imageSize = static_cast(height) * width; - // Diagnostics — log the very first transform() per ONNXYOLO so a + // Diagnostics — log the very first transformInto() per ONNXYOLO so a // future ORT shape-mismatch failure (e.g. /model.12/Concat for // non-stride-32 inputs) points at the offending HxW directly. // One line per session — does not flood DebugView. @@ -124,40 +133,42 @@ namespace ANSCENTER { if (s_loggedFirst != this) { s_loggedFirst = this; ANS_DBG("ONNXYOLO", - "transform[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d", + "transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d", inputImageShape.width, inputImageShape.height, width, height, isClassification ? 1 : 0, mat.cols, mat.rows, isDynamicInputShape ? 1 : 0); } - input_node_dims = { 1, 3, height, width }; - input_tensor_size = 1 * 3 * imageSize; - input_values_handler.resize(input_tensor_size); + outDims = { 1, 3, height, width }; + const size_t tensorSize = static_cast(1) * 3 * imageSize; + outBuffer.resize(tensorSize); std::vector channelMats(channels); for (int c = 0; c < channels; ++c) { channelMats[c] = cv::Mat(height, width, CV_32FC1, - input_values_handler.data() + c * imageSize); + outBuffer.data() + c * imageSize); } cv::split(canvas, channelMats); return Ort::Value::CreateTensor( *memory_info_handler, - input_values_handler.data(), - input_tensor_size, - input_node_dims.data(), - input_node_dims.size()); + outBuffer.data(), + tensorSize, + outDims.data(), + outDims.size()); } - Ort::Value ONNXYOLO::transformBatch(const std::vector& images) + Ort::Value ONNXYOLO::transformBatchInto(const std::vector& images, + std::vector& outBuffer, + std::vector& outDims) { if (images.empty()) - throw std::runtime_error("ONNXYOLO::transformBatch: empty input"); + throw std::runtime_error("ONNXYOLO::transformBatchInto: empty input"); const size_t N = images.size(); // Preprocess all images: letterbox → BGR→RGB → float → /255 - // Check if model is classification (first output has 2 dims: [B, nc]) + // (output_node_dims is immutable post-init — safe to read here.) const bool isClassification = !output_node_dims.empty() && output_node_dims[0].size() == 2; @@ -165,7 +176,7 @@ namespace ANSCENTER { batch.reserve(N); for (const auto& img : images) { if (img.empty()) - throw std::runtime_error("ONNXYOLO::transformBatch: empty image in batch"); + throw std::runtime_error("ONNXYOLO::transformBatchInto: empty image in batch"); // Grayscale → BGR if needed cv::Mat bgrImg; @@ -192,31 +203,54 @@ namespace ANSCENTER { const int width = batch[0].cols; const size_t imageSize = static_cast(height) * width; - input_node_dims = { + outDims = { static_cast(N), 3, static_cast(height), static_cast(width) }; - input_tensor_size = N * 3 * imageSize; - input_values_handler.resize(input_tensor_size); + const size_t tensorSize = N * 3 * imageSize; + outBuffer.resize(tensorSize); - // Pack each image into CHW layout (same as transform() for single image) + // Pack each image into CHW layout (same as transformInto() for single image) for (size_t b = 0; b < N; ++b) { const size_t batchOffset = b * 3 * imageSize; std::vector channelMats(3); for (int c = 0; c < 3; ++c) { channelMats[c] = cv::Mat(height, width, CV_32FC1, - input_values_handler.data() + batchOffset + c * imageSize); + outBuffer.data() + batchOffset + c * imageSize); } cv::split(batch[b], channelMats); } return Ort::Value::CreateTensor( *memory_info_handler, - input_values_handler.data(), - input_tensor_size, - input_node_dims.data(), - input_node_dims.size()); + outBuffer.data(), + tensorSize, + outDims.data(), + outDims.size()); + } + + // ------------------------------------------------------------------ + // transform / transformBatch — pure-virtual overrides required by + // BasicOrtHandler. detect() / detectBatch() do NOT call these (they + // call transformInto / transformBatchInto with per-call buffers for + // thread safety). Kept as thin shims that delegate to the *Into + // variants but write the result into the shared base members + // (input_values_handler / input_node_dims). Any external caller that + // happens to invoke these directly gets the legacy behaviour. + // ------------------------------------------------------------------ + Ort::Value ONNXYOLO::transform(const cv::Mat& mat) + { + Ort::Value v = transformInto(mat, input_values_handler, input_node_dims); + input_tensor_size = input_values_handler.size(); + return v; + } + + Ort::Value ONNXYOLO::transformBatch(const std::vector& images) + { + Ort::Value v = transformBatchInto(images, input_values_handler, input_node_dims); + input_tensor_size = input_values_handler.size(); + return v; } // ------------------------------------------------------------------ @@ -241,7 +275,15 @@ namespace ANSCENTER { if (image.empty()) return {}; - Ort::Value inputTensor = transform(image); + // Per-call backing storage for the input tensor. These MUST live + // until ort_session->Run() returns — the Ort::Value is a non-owning + // view into inputBuffer.data(). Local lifetime guarantees no other + // thread can clobber our buffer mid-Run, which is what enables the + // 4 AI tasks sharing one ONNXYOLO to truly run inference in + // parallel on Intel/OpenVINO EP. + std::vector inputBuffer; + std::vector inputDims; + Ort::Value inputTensor = transformInto(image, inputBuffer, inputDims); auto outputTensors = ort_session->Run( Ort::RunOptions{ nullptr }, @@ -282,9 +324,14 @@ namespace ANSCENTER { } } + // resizedShape is the canvas size we letterboxed into. Pre-option-3 + // this came from input_node_dims[3]/[2] which transform() had just + // overwritten — that's no longer thread-safe (other camera's call + // may have moved on). Use our local dims instead; they hold what + // transformInto() actually produced for THIS image. const cv::Size resizedShape( - static_cast(input_node_dims[3]), - static_cast(input_node_dims[2])); + static_cast(inputDims[3]), + static_cast(inputDims[2])); const size_t numOutputs = outputTensors.size(); @@ -1417,8 +1464,11 @@ namespace ANSCENTER { const size_t N = images.size(); - // Fallback to sequential if model has fixed batch=1 - // (input_node_dims[0] == 1 and not dynamic (-1)) + // Fallback to sequential if the MODEL has fixed batch=1. We read + // input_node_dims here only to inspect the model's static input + // shape — option-3 detect()/detectBatch() no longer overwrite + // input_node_dims, so it retains whatever initialize_handler + // populated from the loaded ONNX graph. if (input_node_dims.size() >= 1 && input_node_dims[0] == 1) { std::vector> results(N); for (size_t i = 0; i < N; ++i) @@ -1433,8 +1483,12 @@ namespace ANSCENTER { for (const auto& img : images) originalSizes.push_back(img.size()); - // Batch preprocess + single inference call - Ort::Value inputTensor = transformBatch(images); + // Per-call backing storage — see detect() for the full thread-safety + // rationale. These outlive ort_session->Run() because they are + // local to this function. + std::vector inputBuffer; + std::vector inputDims; + Ort::Value inputTensor = transformBatchInto(images, inputBuffer, inputDims); auto outputTensors = ort_session->Run( Ort::RunOptions{ nullptr }, @@ -1460,9 +1514,11 @@ namespace ANSCENTER { } } + // Use the per-call dims, not the (now-immutable) member, for the + // same reason as detect(). const cv::Size resizedShape( - static_cast(input_node_dims[3]), - static_cast(input_node_dims[2])); + static_cast(inputDims[3]), + static_cast(inputDims[2])); // Determine task type from output shapes (same logic as detect()) const size_t numOutputs = outputTensors.size(); diff --git a/modules/ANSODEngine/ANSONNXYOLO.h b/modules/ANSODEngine/ANSONNXYOLO.h index 3c51b6f..22239d0 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.h +++ b/modules/ANSODEngine/ANSONNXYOLO.h @@ -178,6 +178,23 @@ namespace ANSCENTER { // Cached model input shape cv::Size inputImageShape; bool isDynamicInputShape{ false }; + + // Per-call preprocess that writes into caller-provided buffer + dims + // instead of the BasicOrtHandler members (input_values_handler / + // input_node_dims). detect() / detectBatch() use these so multiple + // threads sharing one ONNXYOLO instance can preprocess and infer + // concurrently — the Ort input tensor's backing storage is unique + // per call and lives on the caller's stack. Without this, thread B + // could overwrite the shared buffer between thread A's transform() + // and thread A's Run(), causing A's Run() to consume B's pixels and + // return detections that belong to B's frame. Observed on + // Intel/OpenVINO EP where nothing else serializes. + Ort::Value transformInto(const cv::Mat& mat, + std::vector& outBuffer, + std::vector& outDims); + Ort::Value transformBatchInto(const std::vector& images, + std::vector& outBuffer, + std::vector& outDims); }; // ====================================================================