feat(ANSODEngine): run Darknet/YOLO exports ("darknet" family) in ONNXYOLO and RTYOLO

A Darknet/YOLO model exported by hank-ai's darknet_onnx_export (v5.1+) has
two outputs: confs [B,N,nc], already objectness x class probability, and
boxes [B,N,1,4], corners x1,y1,x2,y2 normalized to the input. It trains and
predicts on a stretched square and has no NMS in the graph. ONNXYOLO took
the 4-D boxes tensor for segmentation proto masks and letterboxed the input,
so such a model decoded as garbage; a sealed package named as YOLOV4 went to
YOLOOD, which cannot open an .ansmodel at all.

- ONNXYOLO / RTYOLO: recognise the layout once at load (package family
  "darknet", output names confs/boxes, or their shapes), before the seg
  check; stretch-resize the input (single, batch, and RTYOLO's NV12 path);
  new postprocessDarknet: best class per candidate, threshold, stretch map
  to the original image, clip, class-aware NMS.
- dllmain: a sealed package passed as modelType 1 (YOLOV4) is routed as
  detection (3), i.e. to RTYOLO on NVIDIA and ONNXYOLO elsewhere, in
  CreateANSODHandle, CreateANSODHandleEx and OptimizeModelStr.
- Every other model is unchanged: all new branches are gated on the layout.

Verified on macOS only: ANSONNXYOLO.cpp passes a syntax check, and the new
decode functions, extracted into a standalone harness running the model
through onnxruntime, reproduce Darknet's own detections on dog.jpg. NOT yet
built on Windows, and the TensorRT path has not run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-26 18:43:13 +10:00
co-authored by Claude Opus 5.5
parent cb964907f0
commit 0baca1b20d
6 changed files with 378 additions and 8 deletions
+179 -2
View File
@@ -24,6 +24,7 @@ namespace ANSCENTER {
else {
inputImageShape = cv::Size(640, 640);
}
detectDarknetLayout();
}
ONNXYOLO::ONNXYOLO(const std::string& _onnx_path, EngineType engineType,
@@ -39,6 +40,7 @@ namespace ANSCENTER {
else {
inputImageShape = cv::Size(640, 640);
}
detectDarknetLayout();
}
ONNXYOLO::ONNXYOLO(const void* modelData, size_t modelSize,
@@ -54,6 +56,7 @@ namespace ANSCENTER {
else {
inputImageShape = cv::Size(640, 640);
}
detectDarknetLayout();
}
ONNXYOLO::ONNXYOLO(const void* modelData, size_t modelSize,
@@ -70,6 +73,44 @@ namespace ANSCENTER {
else {
inputImageShape = cv::Size(640, 640);
}
detectDarknetLayout();
}
// ------------------------------------------------------------------
// detectDarknetLayout — recognise a Darknet/YOLO export (hank-ai
// darknet_onnx_export) from its outputs, once, at load time:
// confs [B,N,nc] (objectness x class probability)
// boxes [B,N,1,4] or [B,N,4] (x1,y1,x2,y2 normalised 0..1)
// Matched by name first, then by shape. Must be decided before the
// segmentation check in detect(), which would otherwise take the 4-D
// boxes tensor for proto masks.
// ------------------------------------------------------------------
void ONNXYOLO::detectDarknetLayout()
{
isDarknetLayout = false;
if (num_outputs != 2 || output_node_dims.size() != 2) return;
if (output_node_names_.size() == 2) {
const bool byName =
(output_node_names_[0] == "confs" && output_node_names_[1] == "boxes") ||
(output_node_names_[0] == "boxes" && output_node_names_[1] == "confs");
if (byName) { isDarknetLayout = true; return; }
}
// By shape: exactly one output ends in 4 (the boxes), and both
// carry the same number of candidates N in dim 1.
auto isBoxes = [](const std::vector<int64_t>& d) {
return (d.size() == 4 && d[2] == 1 && d[3] == 4) ||
(d.size() == 3 && d[2] == 4);
};
const auto& d0 = output_node_dims[0];
const auto& d1 = output_node_dims[1];
if (isBoxes(d0) == isBoxes(d1)) return;
const auto& boxes = isBoxes(d0) ? d0 : d1;
const auto& confs = isBoxes(d0) ? d1 : d0;
if (confs.size() != 3 || confs[2] < 1) return;
if (confs[1] <= 0 || confs[1] != boxes[1]) return;
isDarknetLayout = true;
}
// ------------------------------------------------------------------
@@ -143,6 +184,11 @@ namespace ANSCENTER {
// Classification: direct resize (no letterbox padding) — matches ANSONNXCL
cv::resize(bgrMat, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
0, 0, cv::INTER_LINEAR);
} else if (isDarknetLayout) {
// Darknet: stretch the whole frame to the network size (no
// letterbox) — Darknet trains and predicts on a stretched square
cv::resize(bgrMat, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
0, 0, cv::INTER_LINEAR);
} else {
// Detection/Seg/Pose/OBB: Ultralytics letterbox
letterBox(bgrMat, canvas, inputImageShape);
@@ -164,10 +210,10 @@ namespace ANSCENTER {
if (s_loggedFirst != this) {
s_loggedFirst = this;
ANS_DBG("ONNXYOLO",
"transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
"transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d darknet=%d",
inputImageShape.width, inputImageShape.height, width, height,
isClassification ? 1 : 0, mat.cols, mat.rows,
isDynamicInputShape ? 1 : 0);
isDynamicInputShape ? 1 : 0, isDarknetLayout ? 1 : 0);
}
outDims = { 1, 3, height, width };
@@ -222,6 +268,10 @@ namespace ANSCENTER {
// Classification: direct resize (no letterbox)
cv::resize(bgrImg, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
0, 0, cv::INTER_LINEAR);
} else if (isDarknetLayout) {
// Darknet: stretch resize (no letterbox)
cv::resize(bgrImg, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
0, 0, cv::INTER_LINEAR);
} else {
letterBox(bgrImg, canvas, inputImageShape);
}
@@ -288,6 +338,7 @@ namespace ANSCENTER {
// detect — full pipeline with auto task detection
//
// Decision logic:
// darknet layout (load time) → darknet detection
// 2 outputs (second 4D) → segmentation
// 1 output, 2D → classification
// 1 output, 3D end2end dim2=6 → detection
@@ -366,6 +417,12 @@ namespace ANSCENTER {
const size_t numOutputs = outputTensors.size();
// ── Darknet: confs + boxes (checked before seg — boxes is 4-D) ──
if (isDarknetLayout) {
return postprocessDarknet(image.size(), outputTensors, classNames,
confThreshold, iouThreshold);
}
// ── Segmentation: 2 outputs (detections + proto masks) ──────────
if (numOutputs >= 2) {
const auto protoShape = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();
@@ -1392,6 +1449,118 @@ namespace ANSCENTER {
return results;
}
// ====================================================================
// DARKNET — postprocess
//
// hank-ai darknet_onnx_export (v5.1+): confs [B,N,nc] is already
// objectness x class probability; boxes [B,N,1,4] holds corners
// x1,y1,x2,y2 normalised 0..1 to the (stretched) network input. The
// graph has no NMS. Best class per candidate, threshold, map to the
// original image (a stretch, so x*origW / y*origH), clip, then
// class-aware NMS.
// ====================================================================
std::vector<Object> ONNXYOLO::postprocessDarknet(
const cv::Size& originalImageSize,
std::vector<Ort::Value>& outputTensors,
const std::vector<std::string>& classNames,
float confThreshold, float iouThreshold, int maxDet)
{
if (outputTensors.size() < 2) return {};
// Match by name first, then by shape (the boxes tensor ends in 4)
int confsIdx = -1, boxesIdx = -1;
for (size_t t = 0; t < outputTensors.size() && t < output_node_names_.size(); ++t) {
if (output_node_names_[t] == "confs") confsIdx = static_cast<int>(t);
else if (output_node_names_[t] == "boxes") boxesIdx = static_cast<int>(t);
}
if (confsIdx < 0 || boxesIdx < 0) {
confsIdx = boxesIdx = -1;
for (size_t t = 0; t < outputTensors.size(); ++t) {
const auto sh = outputTensors[t].GetTensorTypeAndShapeInfo().GetShape();
if (!sh.empty() && sh.back() == 4 && boxesIdx < 0) boxesIdx = static_cast<int>(t);
else if (confsIdx < 0) confsIdx = static_cast<int>(t);
}
}
if (confsIdx < 0 || boxesIdx < 0) return {};
const auto confsShape = outputTensors[confsIdx].GetTensorTypeAndShapeInfo().GetShape();
const auto boxesInfo = outputTensors[boxesIdx].GetTensorTypeAndShapeInfo();
if (confsShape.size() != 3) return {};
const int numBoxes = static_cast<int>(confsShape[1]);
const int numClasses = static_cast<int>(confsShape[2]);
if (numBoxes <= 0 || numClasses <= 0) return {};
if (boxesInfo.GetElementCount() != static_cast<size_t>(numBoxes) * 4) return {};
const float* confs = outputTensors[confsIdx].GetTensorMutableData<float>();
const float* boxes = outputTensors[boxesIdx].GetTensorMutableData<float>();
const float origW = static_cast<float>(originalImageSize.width);
const float origH = static_cast<float>(originalImageSize.height);
struct Candidate { float x1, y1, x2, y2, conf; int classId; };
std::vector<Candidate> candidates;
for (int i = 0; i < numBoxes; ++i) {
const float* scoresPtr = confs + static_cast<size_t>(i) * numClasses;
float maxScore = scoresPtr[0];
int bestClass = 0;
for (int c = 1; c < numClasses; ++c) {
if (scoresPtr[c] > maxScore) { maxScore = scoresPtr[c]; bestClass = c; }
}
if (maxScore <= confThreshold) continue;
const float* b = boxes + static_cast<size_t>(i) * 4;
float x1 = clamp(b[0] * origW, 0.f, origW);
float y1 = clamp(b[1] * origH, 0.f, origH);
float x2 = clamp(b[2] * origW, 0.f, origW);
float y2 = clamp(b[3] * origH, 0.f, origH);
if (x2 - x1 < 1.f || y2 - y1 < 1.f) continue;
candidates.push_back({ x1, y1, x2, y2, maxScore, bestClass });
}
if (candidates.empty()) return {};
// Class-aware NMS (boxes are already in image pixels, so compare
// class ids rather than offsetting by a fixed MAX_WH)
std::vector<int> sortedIdx(candidates.size());
std::iota(sortedIdx.begin(), sortedIdx.end(), 0);
std::sort(sortedIdx.begin(), sortedIdx.end(),
[&](int a, int b) { return candidates[a].conf > candidates[b].conf; });
std::vector<bool> suppressed(sortedIdx.size(), false);
std::vector<Object> results;
results.reserve(std::min(static_cast<int>(sortedIdx.size()), maxDet));
for (size_t i = 0; i < sortedIdx.size() && static_cast<int>(results.size()) < maxDet; ++i) {
if (suppressed[i]) continue;
const auto& cur = candidates[sortedIdx[i]];
const float curArea = (cur.x2 - cur.x1) * (cur.y2 - cur.y1);
for (size_t j = i + 1; j < sortedIdx.size(); ++j) {
if (suppressed[j]) continue;
const auto& o = candidates[sortedIdx[j]];
if (o.classId != cur.classId) continue;
float iw = std::min(cur.x2, o.x2) - std::max(cur.x1, o.x1);
float ih = std::min(cur.y2, o.y2) - std::max(cur.y1, o.y1);
if (iw <= 0.f || ih <= 0.f) continue;
float inter = iw * ih;
float ua = curArea + (o.x2 - o.x1) * (o.y2 - o.y1) - inter;
if (ua > 0.f && inter / ua > iouThreshold) suppressed[j] = true;
}
Object obj;
obj.classId = cur.classId;
obj.confidence = cur.conf;
obj.box = cv::Rect(static_cast<int>(cur.x1), static_cast<int>(cur.y1),
static_cast<int>(cur.x2 - cur.x1), static_cast<int>(cur.y2 - cur.y1));
if (cur.classId >= 0 && cur.classId < static_cast<int>(classNames.size()))
obj.className = classNames[cur.classId];
results.push_back(std::move(obj));
}
return results;
}
// ====================================================================
// CLASSIFICATION — postprocess
// ====================================================================
@@ -1569,6 +1738,11 @@ namespace ANSCENTER {
}
// Dispatch to correct postprocess method
if (isDarknetLayout) {
results[i] = postprocessDarknet(originalSizes[i], perImageOutputs, classNames,
confThreshold, iouThreshold);
continue;
}
if (numOutputs >= 2) {
const auto protoShape = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();
if (protoShape.size() == 4) {
@@ -1718,6 +1892,8 @@ namespace ANSCENTER {
}
m_ortEngine = std::make_shared<class ONNXYOLO>(
_packageModel.data(), _packageModel.size(), _packageName);
// The manifest's family outranks what the outputs look like.
if (_packageFamily == "darknet") m_ortEngine->setDarknetLayout(true);
return true;
}
if (!FileExist(_modelFilePath)) {
@@ -1744,6 +1920,7 @@ namespace ANSCENTER {
}
m_ortEngine = std::make_shared<class ONNXYOLO>(
_packageModel.data(), _packageModel.size(), _packageName, engineType);
if (_packageFamily == "darknet") m_ortEngine->setDarknetLayout(true);
return true;
}
if (!FileExist(_modelFilePath)) {