fix onnx yolo race condition
This commit is contained in:
@@ -211,7 +211,16 @@
|
||||
"Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\)\")",
|
||||
"Bash(python -c \"import reportlab.pdfbase.pdfmetrics; print\\('pages: \\(pypdf not available — file size confirms\\)'\\)\")",
|
||||
"Bash(python -m pip install pypdf --quiet)",
|
||||
"Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\); print\\('first page text \\(head\\):', r.pages[0].extract_text\\(\\)[:300].replace\\(chr\\(10\\), ' '\\)\\)\")"
|
||||
"Bash(python -c \"from pypdf import PdfReader; r = PdfReader\\(r'C:/Projects/CLionProjects/ANSCORE/ANSODEngine_Tracker_Stabilization_Guide.pdf'\\); print\\('pages:', len\\(r.pages\\)\\); print\\('first page text \\(head\\):', r.pages[0].extract_text\\(\\)[:300].replace\\(chr\\(10\\), ' '\\)\\)\")",
|
||||
"Bash(awk -F'\\\\t' '{print NR}' \"C:/Users/nghia/Downloads/ANSLEGION.log1.log\")",
|
||||
"Bash(awk -F'\\\\t' 'END {print \"lines:\", NR, \" duration_s:\", $2, \" rate_lines/s:\", NR/$2}' \"C:/Users/nghia/Downloads/ANSLEGION.log1.log\")",
|
||||
"Bash(grep -nE \"RunInference|RunInferencesBatch|infer\\\\\\\\\\(\\\\\\\\\\)|set_input_tensor|inference_request|m_inferRequest|mutex|lock_guard\" C:/Projects/CLionProjects/ANSCORE/engines/OpenVINOEngine/src/**/*.cpp C:/Projects/CLionProjects/ANSCORE/engines/OpenVINOEngine/include/**/*.h)",
|
||||
"Bash(grep -nE \"DetectObjects|TryLockWithTimeout|ortRun|->Run\\\\\\\\\\(|m_ortEngine|infer|RunInference\" C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.cpp)",
|
||||
"Bash(grep -nE \"ONNXYOLO::detect|ort_session->Run|set_input|detect\\\\\\\\\\(.*const.*Mat|m_inputBuffer|m_input_tensor|allocator|ortInput|run_options|preprocess|postprocess|class ONNXYOLO\" C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.cpp C:/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/ANSONNXYOLO.h)",
|
||||
"Bash(ninja ANSODEngine)",
|
||||
"PowerShell($env:Path = \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Tools\\\\Llvm\\\\x64\\\\bin;\" + $env:Path; & \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvarsall.bat\" x64 2>&1 | Out-Null; Get-Command ninja -ErrorAction SilentlyContinue | Select-Object Source)",
|
||||
"PowerShell($ninjaPaths = @\\(\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\Common7\\\\IDE\\\\CommonExtensions\\\\Microsoft\\\\CMake\\\\Ninja\\\\ninja.exe\", \"C:\\\\ProgramData\\\\chocolatey\\\\bin\\\\ninja.exe\", \"C:\\\\tools\\\\ninja\\\\ninja.exe\"\\); foreach \\($p in $ninjaPaths\\) { if \\(Test-Path $p\\) { Write-Output $p } })",
|
||||
"PowerShell(cmd /c '\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Auxiliary\\\\Build\\\\vcvarsall.bat\" x64 >nul 2>&1 && cd /d C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release && \"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\Common7\\\\IDE\\\\CommonExtensions\\\\Microsoft\\\\CMake\\\\Ninja\\\\ninja.exe\" ANSODEngine 2>&1')"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,9 +82,15 @@ namespace ANSCENTER {
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// transform — BGR → RGB, letterbox, /255, HWC→CHW
|
||||
// transformInto — BGR → RGB, letterbox, /255, HWC→CHW into a
|
||||
// caller-provided buffer + dims vector. Thread-safe across instances:
|
||||
// each call's tensor backing memory is unique to the caller, so two
|
||||
// threads sharing one ONNXYOLO can preprocess in parallel without
|
||||
// racing on a shared input_values_handler.
|
||||
// ------------------------------------------------------------------
|
||||
Ort::Value ONNXYOLO::transform(const cv::Mat& mat)
|
||||
Ort::Value ONNXYOLO::transformInto(const cv::Mat& mat,
|
||||
std::vector<float>& outBuffer,
|
||||
std::vector<int64_t>& outDims)
|
||||
{
|
||||
// Grayscale → BGR if needed
|
||||
cv::Mat bgrMat;
|
||||
@@ -94,7 +100,10 @@ namespace ANSCENTER {
|
||||
bgrMat = mat;
|
||||
}
|
||||
|
||||
// Check if model is classification (first output has 2 dims: [B, nc])
|
||||
// Check if model is classification (first output has 2 dims: [B, nc]).
|
||||
// output_node_dims is populated once at session creation in
|
||||
// BasicOrtHandler::initialize_handler and never mutated by this
|
||||
// class, so reading it concurrently from multiple threads is safe.
|
||||
const bool isClassification = !output_node_dims.empty()
|
||||
&& output_node_dims[0].size() == 2;
|
||||
|
||||
@@ -116,7 +125,7 @@ namespace ANSCENTER {
|
||||
const int width = canvas.cols;
|
||||
const size_t imageSize = static_cast<size_t>(height) * width;
|
||||
|
||||
// Diagnostics — log the very first transform() per ONNXYOLO so a
|
||||
// Diagnostics — log the very first transformInto() per ONNXYOLO so a
|
||||
// future ORT shape-mismatch failure (e.g. /model.12/Concat for
|
||||
// non-stride-32 inputs) points at the offending HxW directly.
|
||||
// One line per session — does not flood DebugView.
|
||||
@@ -124,40 +133,42 @@ namespace ANSCENTER {
|
||||
if (s_loggedFirst != this) {
|
||||
s_loggedFirst = this;
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"transform[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
|
||||
"transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
|
||||
inputImageShape.width, inputImageShape.height, width, height,
|
||||
isClassification ? 1 : 0, mat.cols, mat.rows,
|
||||
isDynamicInputShape ? 1 : 0);
|
||||
}
|
||||
|
||||
input_node_dims = { 1, 3, height, width };
|
||||
input_tensor_size = 1 * 3 * imageSize;
|
||||
input_values_handler.resize(input_tensor_size);
|
||||
outDims = { 1, 3, height, width };
|
||||
const size_t tensorSize = static_cast<size_t>(1) * 3 * imageSize;
|
||||
outBuffer.resize(tensorSize);
|
||||
|
||||
std::vector<cv::Mat> channelMats(channels);
|
||||
for (int c = 0; c < channels; ++c) {
|
||||
channelMats[c] = cv::Mat(height, width, CV_32FC1,
|
||||
input_values_handler.data() + c * imageSize);
|
||||
outBuffer.data() + c * imageSize);
|
||||
}
|
||||
cv::split(canvas, channelMats);
|
||||
|
||||
return Ort::Value::CreateTensor<float>(
|
||||
*memory_info_handler,
|
||||
input_values_handler.data(),
|
||||
input_tensor_size,
|
||||
input_node_dims.data(),
|
||||
input_node_dims.size());
|
||||
outBuffer.data(),
|
||||
tensorSize,
|
||||
outDims.data(),
|
||||
outDims.size());
|
||||
}
|
||||
|
||||
Ort::Value ONNXYOLO::transformBatch(const std::vector<cv::Mat>& images)
|
||||
Ort::Value ONNXYOLO::transformBatchInto(const std::vector<cv::Mat>& images,
|
||||
std::vector<float>& outBuffer,
|
||||
std::vector<int64_t>& outDims)
|
||||
{
|
||||
if (images.empty())
|
||||
throw std::runtime_error("ONNXYOLO::transformBatch: empty input");
|
||||
throw std::runtime_error("ONNXYOLO::transformBatchInto: empty input");
|
||||
|
||||
const size_t N = images.size();
|
||||
|
||||
// Preprocess all images: letterbox → BGR→RGB → float → /255
|
||||
// Check if model is classification (first output has 2 dims: [B, nc])
|
||||
// (output_node_dims is immutable post-init — safe to read here.)
|
||||
const bool isClassification = !output_node_dims.empty()
|
||||
&& output_node_dims[0].size() == 2;
|
||||
|
||||
@@ -165,7 +176,7 @@ namespace ANSCENTER {
|
||||
batch.reserve(N);
|
||||
for (const auto& img : images) {
|
||||
if (img.empty())
|
||||
throw std::runtime_error("ONNXYOLO::transformBatch: empty image in batch");
|
||||
throw std::runtime_error("ONNXYOLO::transformBatchInto: empty image in batch");
|
||||
|
||||
// Grayscale → BGR if needed
|
||||
cv::Mat bgrImg;
|
||||
@@ -192,31 +203,54 @@ namespace ANSCENTER {
|
||||
const int width = batch[0].cols;
|
||||
const size_t imageSize = static_cast<size_t>(height) * width;
|
||||
|
||||
input_node_dims = {
|
||||
outDims = {
|
||||
static_cast<int64_t>(N), 3,
|
||||
static_cast<int64_t>(height),
|
||||
static_cast<int64_t>(width)
|
||||
};
|
||||
input_tensor_size = N * 3 * imageSize;
|
||||
input_values_handler.resize(input_tensor_size);
|
||||
const size_t tensorSize = N * 3 * imageSize;
|
||||
outBuffer.resize(tensorSize);
|
||||
|
||||
// Pack each image into CHW layout (same as transform() for single image)
|
||||
// Pack each image into CHW layout (same as transformInto() for single image)
|
||||
for (size_t b = 0; b < N; ++b) {
|
||||
const size_t batchOffset = b * 3 * imageSize;
|
||||
std::vector<cv::Mat> channelMats(3);
|
||||
for (int c = 0; c < 3; ++c) {
|
||||
channelMats[c] = cv::Mat(height, width, CV_32FC1,
|
||||
input_values_handler.data() + batchOffset + c * imageSize);
|
||||
outBuffer.data() + batchOffset + c * imageSize);
|
||||
}
|
||||
cv::split(batch[b], channelMats);
|
||||
}
|
||||
|
||||
return Ort::Value::CreateTensor<float>(
|
||||
*memory_info_handler,
|
||||
input_values_handler.data(),
|
||||
input_tensor_size,
|
||||
input_node_dims.data(),
|
||||
input_node_dims.size());
|
||||
outBuffer.data(),
|
||||
tensorSize,
|
||||
outDims.data(),
|
||||
outDims.size());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// transform / transformBatch — pure-virtual overrides required by
|
||||
// BasicOrtHandler. detect() / detectBatch() do NOT call these (they
|
||||
// call transformInto / transformBatchInto with per-call buffers for
|
||||
// thread safety). Kept as thin shims that delegate to the *Into
|
||||
// variants but write the result into the shared base members
|
||||
// (input_values_handler / input_node_dims). Any external caller that
|
||||
// happens to invoke these directly gets the legacy behaviour.
|
||||
// ------------------------------------------------------------------
|
||||
Ort::Value ONNXYOLO::transform(const cv::Mat& mat)
|
||||
{
|
||||
Ort::Value v = transformInto(mat, input_values_handler, input_node_dims);
|
||||
input_tensor_size = input_values_handler.size();
|
||||
return v;
|
||||
}
|
||||
|
||||
Ort::Value ONNXYOLO::transformBatch(const std::vector<cv::Mat>& images)
|
||||
{
|
||||
Ort::Value v = transformBatchInto(images, input_values_handler, input_node_dims);
|
||||
input_tensor_size = input_values_handler.size();
|
||||
return v;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
@@ -241,7 +275,15 @@ namespace ANSCENTER {
|
||||
if (image.empty())
|
||||
return {};
|
||||
|
||||
Ort::Value inputTensor = transform(image);
|
||||
// Per-call backing storage for the input tensor. These MUST live
|
||||
// until ort_session->Run() returns — the Ort::Value is a non-owning
|
||||
// view into inputBuffer.data(). Local lifetime guarantees no other
|
||||
// thread can clobber our buffer mid-Run, which is what enables the
|
||||
// 4 AI tasks sharing one ONNXYOLO to truly run inference in
|
||||
// parallel on Intel/OpenVINO EP.
|
||||
std::vector<float> inputBuffer;
|
||||
std::vector<int64_t> inputDims;
|
||||
Ort::Value inputTensor = transformInto(image, inputBuffer, inputDims);
|
||||
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
@@ -282,9 +324,14 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// resizedShape is the canvas size we letterboxed into. Pre-option-3
|
||||
// this came from input_node_dims[3]/[2] which transform() had just
|
||||
// overwritten — that's no longer thread-safe (other camera's call
|
||||
// may have moved on). Use our local dims instead; they hold what
|
||||
// transformInto() actually produced for THIS image.
|
||||
const cv::Size resizedShape(
|
||||
static_cast<int>(input_node_dims[3]),
|
||||
static_cast<int>(input_node_dims[2]));
|
||||
static_cast<int>(inputDims[3]),
|
||||
static_cast<int>(inputDims[2]));
|
||||
|
||||
const size_t numOutputs = outputTensors.size();
|
||||
|
||||
@@ -1417,8 +1464,11 @@ namespace ANSCENTER {
|
||||
|
||||
const size_t N = images.size();
|
||||
|
||||
// Fallback to sequential if model has fixed batch=1
|
||||
// (input_node_dims[0] == 1 and not dynamic (-1))
|
||||
// Fallback to sequential if the MODEL has fixed batch=1. We read
|
||||
// input_node_dims here only to inspect the model's static input
|
||||
// shape — option-3 detect()/detectBatch() no longer overwrite
|
||||
// input_node_dims, so it retains whatever initialize_handler
|
||||
// populated from the loaded ONNX graph.
|
||||
if (input_node_dims.size() >= 1 && input_node_dims[0] == 1) {
|
||||
std::vector<std::vector<Object>> results(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
@@ -1433,8 +1483,12 @@ namespace ANSCENTER {
|
||||
for (const auto& img : images)
|
||||
originalSizes.push_back(img.size());
|
||||
|
||||
// Batch preprocess + single inference call
|
||||
Ort::Value inputTensor = transformBatch(images);
|
||||
// Per-call backing storage — see detect() for the full thread-safety
|
||||
// rationale. These outlive ort_session->Run() because they are
|
||||
// local to this function.
|
||||
std::vector<float> inputBuffer;
|
||||
std::vector<int64_t> inputDims;
|
||||
Ort::Value inputTensor = transformBatchInto(images, inputBuffer, inputDims);
|
||||
|
||||
auto outputTensors = ort_session->Run(
|
||||
Ort::RunOptions{ nullptr },
|
||||
@@ -1460,9 +1514,11 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// Use the per-call dims, not the (now-immutable) member, for the
|
||||
// same reason as detect().
|
||||
const cv::Size resizedShape(
|
||||
static_cast<int>(input_node_dims[3]),
|
||||
static_cast<int>(input_node_dims[2]));
|
||||
static_cast<int>(inputDims[3]),
|
||||
static_cast<int>(inputDims[2]));
|
||||
|
||||
// Determine task type from output shapes (same logic as detect())
|
||||
const size_t numOutputs = outputTensors.size();
|
||||
|
||||
@@ -178,6 +178,23 @@ namespace ANSCENTER {
|
||||
// Cached model input shape
|
||||
cv::Size inputImageShape;
|
||||
bool isDynamicInputShape{ false };
|
||||
|
||||
// Per-call preprocess that writes into caller-provided buffer + dims
|
||||
// instead of the BasicOrtHandler members (input_values_handler /
|
||||
// input_node_dims). detect() / detectBatch() use these so multiple
|
||||
// threads sharing one ONNXYOLO instance can preprocess and infer
|
||||
// concurrently — the Ort input tensor's backing storage is unique
|
||||
// per call and lives on the caller's stack. Without this, thread B
|
||||
// could overwrite the shared buffer between thread A's transform()
|
||||
// and thread A's Run(), causing A's Run() to consume B's pixels and
|
||||
// return detections that belong to B's frame. Observed on
|
||||
// Intel/OpenVINO EP where nothing else serializes.
|
||||
Ort::Value transformInto(const cv::Mat& mat,
|
||||
std::vector<float>& outBuffer,
|
||||
std::vector<int64_t>& outDims);
|
||||
Ort::Value transformBatchInto(const std::vector<cv::Mat>& images,
|
||||
std::vector<float>& outBuffer,
|
||||
std::vector<int64_t>& outDims);
|
||||
};
|
||||
|
||||
// ====================================================================
|
||||
|
||||
Reference in New Issue
Block a user