fix onnx yolo race condition

This commit is contained in:
2026-05-04 23:45:05 +10:00
parent 03a63142ec
commit 91d81cbd88
3 changed files with 118 additions and 36 deletions
+10 -1
View File
@@ -211,7 +211,16 @@
"Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\)\")", "Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\)\")",
"Bash(python -c \"import reportlab.pdfbase.pdfmetrics; print\\('pages: \\(pypdf not available — file size confirms\\)'\\)\")", "Bash(python -c \"import reportlab.pdfbase.pdfmetrics; print\\('pages: \\(pypdf not available — file size confirms\\)'\\)\")",
"Bash(python -m pip install pypdf --quiet)", "Bash(python -m pip install pypdf --quiet)",
"Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\); print\\('first page text \\(head\\):', r.pages[0].extract_text\\(\\)[:300].replace\\(chr\\(10\\), ' '\\)\\)\")" "Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\); print\\('first page text \\(head\\):', r.pages[0].extract_text\\(\\)[:300].replace\\(chr\\(10\\), ' '\\)\\)\")",
"Bash(awk -F'\\\\t' '{print NR}' \"C:/Users/nghia/Downloads/ANSLEGION.log1.log\")",
"Bash(awk -F'\\\\t' 'END {print \"lines:\", NR, \" duration_s:\", $2, \" rate_lines/s:\", NR/$2}' \"C:/Users/nghia/Downloads/ANSLEGION.log1.log\")",
"Bash(grep -nE \"RunInference|RunInferencesBatch|infer\\\\\\\\\\(\\\\\\\\\\)|set_input_tensor|inference_request|m_inferRequest|mutex|lock_guard\" C:/Projects/CLionProjects/ANSCORE/engines/OpenVINOEngine/src/**/*.cpp C:/Projects/CLionProjects/ANSCORE/engines/OpenVINOEngine/include/**/*.h)",
"Bash(grep -nE \"DetectObjects|TryLockWithTimeout|ortRun|->Run\\\\\\\\\\(|m_ortEngine|infer|RunInference\" C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.cpp)",
"Bash(grep -nE \"ONNXYOLO::detect|ort_session->Run|set_input|detect\\\\\\\\\\(.*const.*Mat|m_inputBuffer|m_input_tensor|allocator|ortInput|run_options|preprocess|postprocess|class ONNXYOLO\" C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.cpp C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.h)",
"Bash(ninja ANSODEngine)",
"PowerShell($env:Path = \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Tools\\\\Llvm\\\\x64\\\\bin;\" + $env:Path; & \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvarsall.bat\" x64 2>&1 | Out-Null; Get-Command ninja -ErrorAction SilentlyContinue | Select-Object Source)",
"PowerShell($ninjaPaths = @\\(\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\Common7\\\\IDE\\\\CommonExtensions\\\\Microsoft\\\\CMake\\\\Ninja\\\\ninja.exe\", \"C:\\\\ProgramData\\\\chocolatey\\\\bin\\\\ninja.exe\", \"C:\\\\tools\\\\ninja\\\\ninja.exe\"\\); foreach \\($p in $ninjaPaths\\) { if \\(Test-Path $p\\) { Write-Output $p } })",
"PowerShell(cmd /c '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvarsall.bat\" x64 >nul 2>&1 && cd /d C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release && \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\Common7\\\\IDE\\\\CommonExtensions\\\\Microsoft\\\\CMake\\\\Ninja\\\\ninja.exe\" ANSODEngine 2>&1')"
] ]
} }
} }
+91 -35
View File
@@ -82,9 +82,15 @@ namespace ANSCENTER {
} }
// ------------------------------------------------------------------ // ------------------------------------------------------------------
// transform — BGR → RGB, letterbox, /255, HWC→CHW // transformInto — BGR → RGB, letterbox, /255, HWC→CHW into a
// caller-provided buffer + dims vector. Thread-safe across instances:
// each call's tensor backing memory is unique to the caller, so two
// threads sharing one ONNXYOLO can preprocess in parallel without
// racing on a shared input_values_handler.
// ------------------------------------------------------------------ // ------------------------------------------------------------------
Ort::Value ONNXYOLO::transform(const cv::Mat& mat) Ort::Value ONNXYOLO::transformInto(const cv::Mat& mat,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims)
{ {
// Grayscale → BGR if needed // Grayscale → BGR if needed
cv::Mat bgrMat; cv::Mat bgrMat;
@@ -94,7 +100,10 @@ namespace ANSCENTER {
bgrMat = mat; bgrMat = mat;
} }
// Check if model is classification (first output has 2 dims: [B, nc]) // Check if model is classification (first output has 2 dims: [B, nc]).
// output_node_dims is populated once at session creation in
// BasicOrtHandler::initialize_handler and never mutated by this
// class, so reading it concurrently from multiple threads is safe.
const bool isClassification = !output_node_dims.empty() const bool isClassification = !output_node_dims.empty()
&& output_node_dims[0].size() == 2; && output_node_dims[0].size() == 2;
@@ -116,7 +125,7 @@ namespace ANSCENTER {
const int width = canvas.cols; const int width = canvas.cols;
const size_t imageSize = static_cast<size_t>(height) * width; const size_t imageSize = static_cast<size_t>(height) * width;
// Diagnostics — log the very first transform() per ONNXYOLO so a // Diagnostics — log the very first transformInto() per ONNXYOLO so a
// future ORT shape-mismatch failure (e.g. /model.12/Concat for // future ORT shape-mismatch failure (e.g. /model.12/Concat for
// non-stride-32 inputs) points at the offending HxW directly. // non-stride-32 inputs) points at the offending HxW directly.
// One line per session — does not flood DebugView. // One line per session — does not flood DebugView.
@@ -124,40 +133,42 @@ namespace ANSCENTER {
if (s_loggedFirst != this) { if (s_loggedFirst != this) {
s_loggedFirst = this; s_loggedFirst = this;
ANS_DBG("ONNXYOLO", ANS_DBG("ONNXYOLO",
"transform[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d", "transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
inputImageShape.width, inputImageShape.height, width, height, inputImageShape.width, inputImageShape.height, width, height,
isClassification ? 1 : 0, mat.cols, mat.rows, isClassification ? 1 : 0, mat.cols, mat.rows,
isDynamicInputShape ? 1 : 0); isDynamicInputShape ? 1 : 0);
} }
input_node_dims = { 1, 3, height, width }; outDims = { 1, 3, height, width };
input_tensor_size = 1 * 3 * imageSize; const size_t tensorSize = static_cast<size_t>(1) * 3 * imageSize;
input_values_handler.resize(input_tensor_size); outBuffer.resize(tensorSize);
std::vector<cv::Mat> channelMats(channels); std::vector<cv::Mat> channelMats(channels);
for (int c = 0; c < channels; ++c) { for (int c = 0; c < channels; ++c) {
channelMats[c] = cv::Mat(height, width, CV_32FC1, channelMats[c] = cv::Mat(height, width, CV_32FC1,
input_values_handler.data() + c * imageSize); outBuffer.data() + c * imageSize);
} }
cv::split(canvas, channelMats); cv::split(canvas, channelMats);
return Ort::Value::CreateTensor<float>( return Ort::Value::CreateTensor<float>(
*memory_info_handler, *memory_info_handler,
input_values_handler.data(), outBuffer.data(),
input_tensor_size, tensorSize,
input_node_dims.data(), outDims.data(),
input_node_dims.size()); outDims.size());
} }
Ort::Value ONNXYOLO::transformBatch(const std::vector<cv::Mat>& images) Ort::Value ONNXYOLO::transformBatchInto(const std::vector<cv::Mat>& images,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims)
{ {
if (images.empty()) if (images.empty())
throw std::runtime_error("ONNXYOLO::transformBatch: empty input"); throw std::runtime_error("ONNXYOLO::transformBatchInto: empty input");
const size_t N = images.size(); const size_t N = images.size();
// Preprocess all images: letterbox → BGR→RGB → float → /255 // Preprocess all images: letterbox → BGR→RGB → float → /255
// Check if model is classification (first output has 2 dims: [B, nc]) // (output_node_dims is immutable post-init — safe to read here.)
const bool isClassification = !output_node_dims.empty() const bool isClassification = !output_node_dims.empty()
&& output_node_dims[0].size() == 2; && output_node_dims[0].size() == 2;
@@ -165,7 +176,7 @@ namespace ANSCENTER {
batch.reserve(N); batch.reserve(N);
for (const auto& img : images) { for (const auto& img : images) {
if (img.empty()) if (img.empty())
throw std::runtime_error("ONNXYOLO::transformBatch: empty image in batch"); throw std::runtime_error("ONNXYOLO::transformBatchInto: empty image in batch");
// Grayscale → BGR if needed // Grayscale → BGR if needed
cv::Mat bgrImg; cv::Mat bgrImg;
@@ -192,31 +203,54 @@ namespace ANSCENTER {
const int width = batch[0].cols; const int width = batch[0].cols;
const size_t imageSize = static_cast<size_t>(height) * width; const size_t imageSize = static_cast<size_t>(height) * width;
input_node_dims = { outDims = {
static_cast<int64_t>(N), 3, static_cast<int64_t>(N), 3,
static_cast<int64_t>(height), static_cast<int64_t>(height),
static_cast<int64_t>(width) static_cast<int64_t>(width)
}; };
input_tensor_size = N * 3 * imageSize; const size_t tensorSize = N * 3 * imageSize;
input_values_handler.resize(input_tensor_size); outBuffer.resize(tensorSize);
// Pack each image into CHW layout (same as transform() for single image) // Pack each image into CHW layout (same as transformInto() for single image)
for (size_t b = 0; b < N; ++b) { for (size_t b = 0; b < N; ++b) {
const size_t batchOffset = b * 3 * imageSize; const size_t batchOffset = b * 3 * imageSize;
std::vector<cv::Mat> channelMats(3); std::vector<cv::Mat> channelMats(3);
for (int c = 0; c < 3; ++c) { for (int c = 0; c < 3; ++c) {
channelMats[c] = cv::Mat(height, width, CV_32FC1, channelMats[c] = cv::Mat(height, width, CV_32FC1,
input_values_handler.data() + batchOffset + c * imageSize); outBuffer.data() + batchOffset + c * imageSize);
} }
cv::split(batch[b], channelMats); cv::split(batch[b], channelMats);
} }
return Ort::Value::CreateTensor<float>( return Ort::Value::CreateTensor<float>(
*memory_info_handler, *memory_info_handler,
input_values_handler.data(), outBuffer.data(),
input_tensor_size, tensorSize,
input_node_dims.data(), outDims.data(),
input_node_dims.size()); outDims.size());
}
// ------------------------------------------------------------------
// transform / transformBatch — pure-virtual overrides required by
// BasicOrtHandler. detect() / detectBatch() do NOT call these (they
// call transformInto / transformBatchInto with per-call buffers for
// thread safety). Kept as thin shims that delegate to the *Into
// variants but write the result into the shared base members
// (input_values_handler / input_node_dims). Any external caller that
// happens to invoke these directly gets the legacy behaviour.
// ------------------------------------------------------------------
Ort::Value ONNXYOLO::transform(const cv::Mat& mat)
{
Ort::Value v = transformInto(mat, input_values_handler, input_node_dims);
input_tensor_size = input_values_handler.size();
return v;
}
Ort::Value ONNXYOLO::transformBatch(const std::vector<cv::Mat>& images)
{
Ort::Value v = transformBatchInto(images, input_values_handler, input_node_dims);
input_tensor_size = input_values_handler.size();
return v;
} }
// ------------------------------------------------------------------ // ------------------------------------------------------------------
@@ -241,7 +275,15 @@ namespace ANSCENTER {
if (image.empty()) if (image.empty())
return {}; return {};
Ort::Value inputTensor = transform(image); // Per-call backing storage for the input tensor. These MUST live
// until ort_session->Run() returns — the Ort::Value is a non-owning
// view into inputBuffer.data(). Local lifetime guarantees no other
// thread can clobber our buffer mid-Run, which is what enables the
// 4 AI tasks sharing one ONNXYOLO to truly run inference in
// parallel on Intel/OpenVINO EP.
std::vector<float> inputBuffer;
std::vector<int64_t> inputDims;
Ort::Value inputTensor = transformInto(image, inputBuffer, inputDims);
auto outputTensors = ort_session->Run( auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr }, Ort::RunOptions{ nullptr },
@@ -282,9 +324,14 @@ namespace ANSCENTER {
} }
} }
// resizedShape is the canvas size we letterboxed into. Pre-option-3
// this came from input_node_dims[3]/[2] which transform() had just
// overwritten — that's no longer thread-safe (other camera's call
// may have moved on). Use our local dims instead; they hold what
// transformInto() actually produced for THIS image.
const cv::Size resizedShape( const cv::Size resizedShape(
static_cast<int>(input_node_dims[3]), static_cast<int>(inputDims[3]),
static_cast<int>(input_node_dims[2])); static_cast<int>(inputDims[2]));
const size_t numOutputs = outputTensors.size(); const size_t numOutputs = outputTensors.size();
@@ -1417,8 +1464,11 @@ namespace ANSCENTER {
const size_t N = images.size(); const size_t N = images.size();
// Fallback to sequential if model has fixed batch=1 // Fallback to sequential if the MODEL has fixed batch=1. We read
// (input_node_dims[0] == 1 and not dynamic (-1)) // input_node_dims here only to inspect the model's static input
// shape — option-3 detect()/detectBatch() no longer overwrite
// input_node_dims, so it retains whatever initialize_handler
// populated from the loaded ONNX graph.
if (input_node_dims.size() >= 1 && input_node_dims[0] == 1) { if (input_node_dims.size() >= 1 && input_node_dims[0] == 1) {
std::vector<std::vector<Object>> results(N); std::vector<std::vector<Object>> results(N);
for (size_t i = 0; i < N; ++i) for (size_t i = 0; i < N; ++i)
@@ -1433,8 +1483,12 @@ namespace ANSCENTER {
for (const auto& img : images) for (const auto& img : images)
originalSizes.push_back(img.size()); originalSizes.push_back(img.size());
// Batch preprocess + single inference call // Per-call backing storage — see detect() for the full thread-safety
Ort::Value inputTensor = transformBatch(images); // rationale. These outlive ort_session->Run() because they are
// local to this function.
std::vector<float> inputBuffer;
std::vector<int64_t> inputDims;
Ort::Value inputTensor = transformBatchInto(images, inputBuffer, inputDims);
auto outputTensors = ort_session->Run( auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr }, Ort::RunOptions{ nullptr },
@@ -1460,9 +1514,11 @@ namespace ANSCENTER {
} }
} }
// Use the per-call dims, not the (now-immutable) member, for the
// same reason as detect().
const cv::Size resizedShape( const cv::Size resizedShape(
static_cast<int>(input_node_dims[3]), static_cast<int>(inputDims[3]),
static_cast<int>(input_node_dims[2])); static_cast<int>(inputDims[2]));
// Determine task type from output shapes (same logic as detect()) // Determine task type from output shapes (same logic as detect())
const size_t numOutputs = outputTensors.size(); const size_t numOutputs = outputTensors.size();
+17
View File
@@ -178,6 +178,23 @@ namespace ANSCENTER {
// Cached model input shape // Cached model input shape
cv::Size inputImageShape; cv::Size inputImageShape;
bool isDynamicInputShape{ false }; bool isDynamicInputShape{ false };
// Per-call preprocess that writes into caller-provided buffer + dims
// instead of the BasicOrtHandler members (input_values_handler /
// input_node_dims). detect() / detectBatch() use these so multiple
// threads sharing one ONNXYOLO instance can preprocess and infer
// concurrently — the Ort input tensor's backing storage is unique
// per call and lives on the caller's stack. Without this, thread B
// could overwrite the shared buffer between thread A's transform()
// and thread A's Run(), causing A's Run() to consume B's pixels and
// return detections that belong to B's frame. Observed on
// Intel/OpenVINO EP where nothing else serializes.
Ort::Value transformInto(const cv::Mat& mat,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
Ort::Value transformBatchInto(const std::vector<cv::Mat>& images,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
}; };
// ==================================================================== // ====================================================================