fix onnx yolo race condition

This commit is contained in:
2026-05-04 23:45:05 +10:00
parent 03a63142ec
commit 91d81cbd88
3 changed files with 118 additions and 36 deletions
+91 -35
View File
@@ -82,9 +82,15 @@ namespace ANSCENTER {
}
// ------------------------------------------------------------------
// transform — BGR → RGB, letterbox, /255, HWC→CHW
// transformInto — BGR → RGB, letterbox, /255, HWC→CHW into a
// caller-provided buffer + dims vector. Thread-safe across instances:
// each call's tensor backing memory is unique to the caller, so two
// threads sharing one ONNXYOLO can preprocess in parallel without
// racing on a shared input_values_handler.
// ------------------------------------------------------------------
Ort::Value ONNXYOLO::transform(const cv::Mat& mat)
Ort::Value ONNXYOLO::transformInto(const cv::Mat& mat,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims)
{
// Grayscale → BGR if needed
cv::Mat bgrMat;
@@ -94,7 +100,10 @@ namespace ANSCENTER {
bgrMat = mat;
}
// Check if model is classification (first output has 2 dims: [B, nc])
// Check if model is classification (first output has 2 dims: [B, nc]).
// output_node_dims is populated once at session creation in
// BasicOrtHandler::initialize_handler and never mutated by this
// class, so reading it concurrently from multiple threads is safe.
const bool isClassification = !output_node_dims.empty()
&& output_node_dims[0].size() == 2;
@@ -116,7 +125,7 @@ namespace ANSCENTER {
const int width = canvas.cols;
const size_t imageSize = static_cast<size_t>(height) * width;
// Diagnostics — log the very first transform() per ONNXYOLO so a
// Diagnostics — log the very first transformInto() per ONNXYOLO so a
// future ORT shape-mismatch failure (e.g. /model.12/Concat for
// non-stride-32 inputs) points at the offending HxW directly.
// One line per session — does not flood DebugView.
@@ -124,40 +133,42 @@ namespace ANSCENTER {
if (s_loggedFirst != this) {
s_loggedFirst = this;
ANS_DBG("ONNXYOLO",
"transform[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
"transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
inputImageShape.width, inputImageShape.height, width, height,
isClassification ? 1 : 0, mat.cols, mat.rows,
isDynamicInputShape ? 1 : 0);
}
input_node_dims = { 1, 3, height, width };
input_tensor_size = 1 * 3 * imageSize;
input_values_handler.resize(input_tensor_size);
outDims = { 1, 3, height, width };
const size_t tensorSize = static_cast<size_t>(1) * 3 * imageSize;
outBuffer.resize(tensorSize);
std::vector<cv::Mat> channelMats(channels);
for (int c = 0; c < channels; ++c) {
channelMats[c] = cv::Mat(height, width, CV_32FC1,
input_values_handler.data() + c * imageSize);
outBuffer.data() + c * imageSize);
}
cv::split(canvas, channelMats);
return Ort::Value::CreateTensor<float>(
*memory_info_handler,
input_values_handler.data(),
input_tensor_size,
input_node_dims.data(),
input_node_dims.size());
outBuffer.data(),
tensorSize,
outDims.data(),
outDims.size());
}
Ort::Value ONNXYOLO::transformBatch(const std::vector<cv::Mat>& images)
Ort::Value ONNXYOLO::transformBatchInto(const std::vector<cv::Mat>& images,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims)
{
if (images.empty())
throw std::runtime_error("ONNXYOLO::transformBatch: empty input");
throw std::runtime_error("ONNXYOLO::transformBatchInto: empty input");
const size_t N = images.size();
// Preprocess all images: letterbox → BGR→RGB → float → /255
// Check if model is classification (first output has 2 dims: [B, nc])
// (output_node_dims is immutable post-init — safe to read here.)
const bool isClassification = !output_node_dims.empty()
&& output_node_dims[0].size() == 2;
@@ -165,7 +176,7 @@ namespace ANSCENTER {
batch.reserve(N);
for (const auto& img : images) {
if (img.empty())
throw std::runtime_error("ONNXYOLO::transformBatch: empty image in batch");
throw std::runtime_error("ONNXYOLO::transformBatchInto: empty image in batch");
// Grayscale → BGR if needed
cv::Mat bgrImg;
@@ -192,31 +203,54 @@ namespace ANSCENTER {
const int width = batch[0].cols;
const size_t imageSize = static_cast<size_t>(height) * width;
input_node_dims = {
outDims = {
static_cast<int64_t>(N), 3,
static_cast<int64_t>(height),
static_cast<int64_t>(width)
};
input_tensor_size = N * 3 * imageSize;
input_values_handler.resize(input_tensor_size);
const size_t tensorSize = N * 3 * imageSize;
outBuffer.resize(tensorSize);
// Pack each image into CHW layout (same as transform() for single image)
// Pack each image into CHW layout (same as transformInto() for single image)
for (size_t b = 0; b < N; ++b) {
const size_t batchOffset = b * 3 * imageSize;
std::vector<cv::Mat> channelMats(3);
for (int c = 0; c < 3; ++c) {
channelMats[c] = cv::Mat(height, width, CV_32FC1,
input_values_handler.data() + batchOffset + c * imageSize);
outBuffer.data() + batchOffset + c * imageSize);
}
cv::split(batch[b], channelMats);
}
return Ort::Value::CreateTensor<float>(
*memory_info_handler,
input_values_handler.data(),
input_tensor_size,
input_node_dims.data(),
input_node_dims.size());
outBuffer.data(),
tensorSize,
outDims.data(),
outDims.size());
}
// ------------------------------------------------------------------
// transform / transformBatch — pure-virtual overrides required by
// BasicOrtHandler. detect() / detectBatch() do NOT call these (they
// call transformInto / transformBatchInto with per-call buffers for
// thread safety). Kept as thin shims that delegate to the *Into
// variants but write the result into the shared base members
// (input_values_handler / input_node_dims). Any external caller that
// happens to invoke these directly gets the legacy behaviour.
// ------------------------------------------------------------------
Ort::Value ONNXYOLO::transform(const cv::Mat& mat)
{
Ort::Value v = transformInto(mat, input_values_handler, input_node_dims);
input_tensor_size = input_values_handler.size();
return v;
}
Ort::Value ONNXYOLO::transformBatch(const std::vector<cv::Mat>& images)
{
Ort::Value v = transformBatchInto(images, input_values_handler, input_node_dims);
input_tensor_size = input_values_handler.size();
return v;
}
// ------------------------------------------------------------------
@@ -241,7 +275,15 @@ namespace ANSCENTER {
if (image.empty())
return {};
Ort::Value inputTensor = transform(image);
// Per-call backing storage for the input tensor. These MUST live
// until ort_session->Run() returns — the Ort::Value is a non-owning
// view into inputBuffer.data(). Local lifetime guarantees no other
// thread can clobber our buffer mid-Run, which is what enables the
// 4 AI tasks sharing one ONNXYOLO to truly run inference in
// parallel on Intel/OpenVINO EP.
std::vector<float> inputBuffer;
std::vector<int64_t> inputDims;
Ort::Value inputTensor = transformInto(image, inputBuffer, inputDims);
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
@@ -282,9 +324,14 @@ namespace ANSCENTER {
}
}
// resizedShape is the canvas size we letterboxed into. Pre-option-3
// this came from input_node_dims[3]/[2] which transform() had just
// overwritten — that's no longer thread-safe (other camera's call
// may have moved on). Use our local dims instead; they hold what
// transformInto() actually produced for THIS image.
const cv::Size resizedShape(
static_cast<int>(input_node_dims[3]),
static_cast<int>(input_node_dims[2]));
static_cast<int>(inputDims[3]),
static_cast<int>(inputDims[2]));
const size_t numOutputs = outputTensors.size();
@@ -1417,8 +1464,11 @@ namespace ANSCENTER {
const size_t N = images.size();
// Fallback to sequential if model has fixed batch=1
// (input_node_dims[0] == 1 and not dynamic (-1))
// Fallback to sequential if the MODEL has fixed batch=1. We read
// input_node_dims here only to inspect the model's static input
// shape — option-3 detect()/detectBatch() no longer overwrite
// input_node_dims, so it retains whatever initialize_handler
// populated from the loaded ONNX graph.
if (input_node_dims.size() >= 1 && input_node_dims[0] == 1) {
std::vector<std::vector<Object>> results(N);
for (size_t i = 0; i < N; ++i)
@@ -1433,8 +1483,12 @@ namespace ANSCENTER {
for (const auto& img : images)
originalSizes.push_back(img.size());
// Batch preprocess + single inference call
Ort::Value inputTensor = transformBatch(images);
// Per-call backing storage — see detect() for the full thread-safety
// rationale. These outlive ort_session->Run() because they are
// local to this function.
std::vector<float> inputBuffer;
std::vector<int64_t> inputDims;
Ort::Value inputTensor = transformBatchInto(images, inputBuffer, inputDims);
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
@@ -1460,9 +1514,11 @@ namespace ANSCENTER {
}
}
// Use the per-call dims, not the (now-immutable) member, for the
// same reason as detect().
const cv::Size resizedShape(
static_cast<int>(input_node_dims[3]),
static_cast<int>(input_node_dims[2]));
static_cast<int>(inputDims[3]),
static_cast<int>(inputDims[2]));
// Determine task type from output shapes (same logic as detect())
const size_t numOutputs = outputTensors.size();
+17
View File
@@ -178,6 +178,23 @@ namespace ANSCENTER {
// Cached model input shape
cv::Size inputImageShape;
bool isDynamicInputShape{ false };
// Per-call preprocess that writes into caller-provided buffer + dims
// instead of the BasicOrtHandler members (input_values_handler /
// input_node_dims). detect() / detectBatch() use these so multiple
// threads sharing one ONNXYOLO instance can preprocess and infer
// concurrently — the Ort input tensor's backing storage is unique
// per call and lives on the caller's stack. Without this, thread B
// could overwrite the shared buffer between thread A's transform()
// and thread A's Run(), causing A's Run() to consume B's pixels and
// return detections that belong to B's frame. Observed on
// Intel/OpenVINO EP where nothing else serializes.
Ort::Value transformInto(const cv::Mat& mat,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
Ort::Value transformBatchInto(const std::vector<cv::Mat>& images,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims);
};
// ====================================================================