feat(ANSODEngine): run Darknet/YOLO exports ("darknet" family) in ONNXYOLO and RTYOLO
A Darknet/YOLO model exported by hank-ai's darknet_onnx_export (v5.1+) has two outputs: confs [B,N,nc], already objectness x class probability, and boxes [B,N,1,4], corners x1,y1,x2,y2 normalized to the input. It trains and predicts on a stretched square and has no NMS in the graph. ONNXYOLO took the 4-D boxes tensor for segmentation proto masks and letterboxed the input, so such a model decoded as garbage; a sealed package named as YOLOV4 went to YOLOOD, which cannot open an .ansmodel at all. - ONNXYOLO / RTYOLO: recognise the layout once at load (package family "darknet", output names confs/boxes, or their shapes), before the seg check; stretch-resize the input (single, batch, and RTYOLO's NV12 path); new postprocessDarknet: best class per candidate, threshold, stretch map to the original image, clip, class-aware NMS. - dllmain: a sealed package passed as modelType 1 (YOLOV4) is routed as detection (3), i.e. to RTYOLO on NVIDIA and ONNXYOLO elsewhere, in CreateANSODHandle, CreateANSODHandleEx and OptimizeModelStr. - Every other model is unchanged: all new branches are gated on the layout. Verified on macOS only: ANSONNXYOLO.cpp passes a syntax check, and the new decode functions, extracted into a standalone harness running the model through onnxruntime, reproduce Darknet's own detections on dog.jpg. NOT yet built on Windows, and the TensorRT path has not run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -24,6 +24,7 @@ namespace ANSCENTER {
|
||||
else {
|
||||
inputImageShape = cv::Size(640, 640);
|
||||
}
|
||||
detectDarknetLayout();
|
||||
}
|
||||
|
||||
ONNXYOLO::ONNXYOLO(const std::string& _onnx_path, EngineType engineType,
|
||||
@@ -39,6 +40,7 @@ namespace ANSCENTER {
|
||||
else {
|
||||
inputImageShape = cv::Size(640, 640);
|
||||
}
|
||||
detectDarknetLayout();
|
||||
}
|
||||
|
||||
ONNXYOLO::ONNXYOLO(const void* modelData, size_t modelSize,
|
||||
@@ -54,6 +56,7 @@ namespace ANSCENTER {
|
||||
else {
|
||||
inputImageShape = cv::Size(640, 640);
|
||||
}
|
||||
detectDarknetLayout();
|
||||
}
|
||||
|
||||
ONNXYOLO::ONNXYOLO(const void* modelData, size_t modelSize,
|
||||
@@ -70,6 +73,44 @@ namespace ANSCENTER {
|
||||
else {
|
||||
inputImageShape = cv::Size(640, 640);
|
||||
}
|
||||
detectDarknetLayout();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// detectDarknetLayout — recognise a Darknet/YOLO export (hank-ai
|
||||
// darknet_onnx_export) from its outputs, once, at load time:
|
||||
// confs [B,N,nc] (objectness x class probability)
|
||||
// boxes [B,N,1,4] or [B,N,4] (x1,y1,x2,y2 normalised 0..1)
|
||||
// Matched by name first, then by shape. Must be decided before the
|
||||
// segmentation check in detect(), which would otherwise take the 4-D
|
||||
// boxes tensor for proto masks.
|
||||
// ------------------------------------------------------------------
|
||||
void ONNXYOLO::detectDarknetLayout()
|
||||
{
|
||||
isDarknetLayout = false;
|
||||
if (num_outputs != 2 || output_node_dims.size() != 2) return;
|
||||
|
||||
if (output_node_names_.size() == 2) {
|
||||
const bool byName =
|
||||
(output_node_names_[0] == "confs" && output_node_names_[1] == "boxes") ||
|
||||
(output_node_names_[0] == "boxes" && output_node_names_[1] == "confs");
|
||||
if (byName) { isDarknetLayout = true; return; }
|
||||
}
|
||||
|
||||
// By shape: exactly one output ends in 4 (the boxes), and both
|
||||
// carry the same number of candidates N in dim 1.
|
||||
auto isBoxes = [](const std::vector<int64_t>& d) {
|
||||
return (d.size() == 4 && d[2] == 1 && d[3] == 4) ||
|
||||
(d.size() == 3 && d[2] == 4);
|
||||
};
|
||||
const auto& d0 = output_node_dims[0];
|
||||
const auto& d1 = output_node_dims[1];
|
||||
if (isBoxes(d0) == isBoxes(d1)) return;
|
||||
const auto& boxes = isBoxes(d0) ? d0 : d1;
|
||||
const auto& confs = isBoxes(d0) ? d1 : d0;
|
||||
if (confs.size() != 3 || confs[2] < 1) return;
|
||||
if (confs[1] <= 0 || confs[1] != boxes[1]) return;
|
||||
isDarknetLayout = true;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
@@ -143,6 +184,11 @@ namespace ANSCENTER {
|
||||
// Classification: direct resize (no letterbox padding) — matches ANSONNXCL
|
||||
cv::resize(bgrMat, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
|
||||
0, 0, cv::INTER_LINEAR);
|
||||
} else if (isDarknetLayout) {
|
||||
// Darknet: stretch the whole frame to the network size (no
|
||||
// letterbox) — Darknet trains and predicts on a stretched square
|
||||
cv::resize(bgrMat, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
|
||||
0, 0, cv::INTER_LINEAR);
|
||||
} else {
|
||||
// Detection/Seg/Pose/OBB: Ultralytics letterbox
|
||||
letterBox(bgrMat, canvas, inputImageShape);
|
||||
@@ -164,10 +210,10 @@ namespace ANSCENTER {
|
||||
if (s_loggedFirst != this) {
|
||||
s_loggedFirst = this;
|
||||
ANS_DBG("ONNXYOLO",
|
||||
"transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d",
|
||||
"transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d darknet=%d",
|
||||
inputImageShape.width, inputImageShape.height, width, height,
|
||||
isClassification ? 1 : 0, mat.cols, mat.rows,
|
||||
isDynamicInputShape ? 1 : 0);
|
||||
isDynamicInputShape ? 1 : 0, isDarknetLayout ? 1 : 0);
|
||||
}
|
||||
|
||||
outDims = { 1, 3, height, width };
|
||||
@@ -222,6 +268,10 @@ namespace ANSCENTER {
|
||||
// Classification: direct resize (no letterbox)
|
||||
cv::resize(bgrImg, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
|
||||
0, 0, cv::INTER_LINEAR);
|
||||
} else if (isDarknetLayout) {
|
||||
// Darknet: stretch resize (no letterbox)
|
||||
cv::resize(bgrImg, canvas, cv::Size(inputImageShape.width, inputImageShape.height),
|
||||
0, 0, cv::INTER_LINEAR);
|
||||
} else {
|
||||
letterBox(bgrImg, canvas, inputImageShape);
|
||||
}
|
||||
@@ -288,6 +338,7 @@ namespace ANSCENTER {
|
||||
// detect — full pipeline with auto task detection
|
||||
//
|
||||
// Decision logic:
|
||||
// darknet layout (load time) → darknet detection
|
||||
// 2 outputs (second 4D) → segmentation
|
||||
// 1 output, 2D → classification
|
||||
// 1 output, 3D end2end dim2=6 → detection
|
||||
@@ -366,6 +417,12 @@ namespace ANSCENTER {
|
||||
|
||||
const size_t numOutputs = outputTensors.size();
|
||||
|
||||
// ── Darknet: confs + boxes (checked before seg — boxes is 4-D) ──
|
||||
if (isDarknetLayout) {
|
||||
return postprocessDarknet(image.size(), outputTensors, classNames,
|
||||
confThreshold, iouThreshold);
|
||||
}
|
||||
|
||||
// ── Segmentation: 2 outputs (detections + proto masks) ──────────
|
||||
if (numOutputs >= 2) {
|
||||
const auto protoShape = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();
|
||||
@@ -1392,6 +1449,118 @@ namespace ANSCENTER {
|
||||
return results;
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// DARKNET — postprocess
|
||||
//
|
||||
// hank-ai darknet_onnx_export (v5.1+): confs [B,N,nc] is already
|
||||
// objectness x class probability; boxes [B,N,1,4] holds corners
|
||||
// x1,y1,x2,y2 normalised 0..1 to the (stretched) network input. The
|
||||
// graph has no NMS. Best class per candidate, threshold, map to the
|
||||
// original image (a stretch, so x*origW / y*origH), clip, then
|
||||
// class-aware NMS.
|
||||
// ====================================================================
|
||||
|
||||
std::vector<Object> ONNXYOLO::postprocessDarknet(
|
||||
const cv::Size& originalImageSize,
|
||||
std::vector<Ort::Value>& outputTensors,
|
||||
const std::vector<std::string>& classNames,
|
||||
float confThreshold, float iouThreshold, int maxDet)
|
||||
{
|
||||
if (outputTensors.size() < 2) return {};
|
||||
|
||||
// Match by name first, then by shape (the boxes tensor ends in 4)
|
||||
int confsIdx = -1, boxesIdx = -1;
|
||||
for (size_t t = 0; t < outputTensors.size() && t < output_node_names_.size(); ++t) {
|
||||
if (output_node_names_[t] == "confs") confsIdx = static_cast<int>(t);
|
||||
else if (output_node_names_[t] == "boxes") boxesIdx = static_cast<int>(t);
|
||||
}
|
||||
if (confsIdx < 0 || boxesIdx < 0) {
|
||||
confsIdx = boxesIdx = -1;
|
||||
for (size_t t = 0; t < outputTensors.size(); ++t) {
|
||||
const auto sh = outputTensors[t].GetTensorTypeAndShapeInfo().GetShape();
|
||||
if (!sh.empty() && sh.back() == 4 && boxesIdx < 0) boxesIdx = static_cast<int>(t);
|
||||
else if (confsIdx < 0) confsIdx = static_cast<int>(t);
|
||||
}
|
||||
}
|
||||
if (confsIdx < 0 || boxesIdx < 0) return {};
|
||||
|
||||
const auto confsShape = outputTensors[confsIdx].GetTensorTypeAndShapeInfo().GetShape();
|
||||
const auto boxesInfo = outputTensors[boxesIdx].GetTensorTypeAndShapeInfo();
|
||||
if (confsShape.size() != 3) return {};
|
||||
|
||||
const int numBoxes = static_cast<int>(confsShape[1]);
|
||||
const int numClasses = static_cast<int>(confsShape[2]);
|
||||
if (numBoxes <= 0 || numClasses <= 0) return {};
|
||||
if (boxesInfo.GetElementCount() != static_cast<size_t>(numBoxes) * 4) return {};
|
||||
|
||||
const float* confs = outputTensors[confsIdx].GetTensorMutableData<float>();
|
||||
const float* boxes = outputTensors[boxesIdx].GetTensorMutableData<float>();
|
||||
|
||||
const float origW = static_cast<float>(originalImageSize.width);
|
||||
const float origH = static_cast<float>(originalImageSize.height);
|
||||
|
||||
struct Candidate { float x1, y1, x2, y2, conf; int classId; };
|
||||
std::vector<Candidate> candidates;
|
||||
|
||||
for (int i = 0; i < numBoxes; ++i) {
|
||||
const float* scoresPtr = confs + static_cast<size_t>(i) * numClasses;
|
||||
float maxScore = scoresPtr[0];
|
||||
int bestClass = 0;
|
||||
for (int c = 1; c < numClasses; ++c) {
|
||||
if (scoresPtr[c] > maxScore) { maxScore = scoresPtr[c]; bestClass = c; }
|
||||
}
|
||||
if (maxScore <= confThreshold) continue;
|
||||
|
||||
const float* b = boxes + static_cast<size_t>(i) * 4;
|
||||
float x1 = clamp(b[0] * origW, 0.f, origW);
|
||||
float y1 = clamp(b[1] * origH, 0.f, origH);
|
||||
float x2 = clamp(b[2] * origW, 0.f, origW);
|
||||
float y2 = clamp(b[3] * origH, 0.f, origH);
|
||||
if (x2 - x1 < 1.f || y2 - y1 < 1.f) continue;
|
||||
candidates.push_back({ x1, y1, x2, y2, maxScore, bestClass });
|
||||
}
|
||||
if (candidates.empty()) return {};
|
||||
|
||||
// Class-aware NMS (boxes are already in image pixels, so compare
|
||||
// class ids rather than offsetting by a fixed MAX_WH)
|
||||
std::vector<int> sortedIdx(candidates.size());
|
||||
std::iota(sortedIdx.begin(), sortedIdx.end(), 0);
|
||||
std::sort(sortedIdx.begin(), sortedIdx.end(),
|
||||
[&](int a, int b) { return candidates[a].conf > candidates[b].conf; });
|
||||
|
||||
std::vector<bool> suppressed(sortedIdx.size(), false);
|
||||
std::vector<Object> results;
|
||||
results.reserve(std::min(static_cast<int>(sortedIdx.size()), maxDet));
|
||||
|
||||
for (size_t i = 0; i < sortedIdx.size() && static_cast<int>(results.size()) < maxDet; ++i) {
|
||||
if (suppressed[i]) continue;
|
||||
const auto& cur = candidates[sortedIdx[i]];
|
||||
const float curArea = (cur.x2 - cur.x1) * (cur.y2 - cur.y1);
|
||||
|
||||
for (size_t j = i + 1; j < sortedIdx.size(); ++j) {
|
||||
if (suppressed[j]) continue;
|
||||
const auto& o = candidates[sortedIdx[j]];
|
||||
if (o.classId != cur.classId) continue;
|
||||
float iw = std::min(cur.x2, o.x2) - std::max(cur.x1, o.x1);
|
||||
float ih = std::min(cur.y2, o.y2) - std::max(cur.y1, o.y1);
|
||||
if (iw <= 0.f || ih <= 0.f) continue;
|
||||
float inter = iw * ih;
|
||||
float ua = curArea + (o.x2 - o.x1) * (o.y2 - o.y1) - inter;
|
||||
if (ua > 0.f && inter / ua > iouThreshold) suppressed[j] = true;
|
||||
}
|
||||
|
||||
Object obj;
|
||||
obj.classId = cur.classId;
|
||||
obj.confidence = cur.conf;
|
||||
obj.box = cv::Rect(static_cast<int>(cur.x1), static_cast<int>(cur.y1),
|
||||
static_cast<int>(cur.x2 - cur.x1), static_cast<int>(cur.y2 - cur.y1));
|
||||
if (cur.classId >= 0 && cur.classId < static_cast<int>(classNames.size()))
|
||||
obj.className = classNames[cur.classId];
|
||||
results.push_back(std::move(obj));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// CLASSIFICATION — postprocess
|
||||
// ====================================================================
|
||||
@@ -1569,6 +1738,11 @@ namespace ANSCENTER {
|
||||
}
|
||||
|
||||
// Dispatch to correct postprocess method
|
||||
if (isDarknetLayout) {
|
||||
results[i] = postprocessDarknet(originalSizes[i], perImageOutputs, classNames,
|
||||
confThreshold, iouThreshold);
|
||||
continue;
|
||||
}
|
||||
if (numOutputs >= 2) {
|
||||
const auto protoShape = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();
|
||||
if (protoShape.size() == 4) {
|
||||
@@ -1718,6 +1892,8 @@ namespace ANSCENTER {
|
||||
}
|
||||
m_ortEngine = std::make_shared<class ONNXYOLO>(
|
||||
_packageModel.data(), _packageModel.size(), _packageName);
|
||||
// The manifest's family outranks what the outputs look like.
|
||||
if (_packageFamily == "darknet") m_ortEngine->setDarknetLayout(true);
|
||||
return true;
|
||||
}
|
||||
if (!FileExist(_modelFilePath)) {
|
||||
@@ -1744,6 +1920,7 @@ namespace ANSCENTER {
|
||||
}
|
||||
m_ortEngine = std::make_shared<class ONNXYOLO>(
|
||||
_packageModel.data(), _packageModel.size(), _packageName, engineType);
|
||||
if (_packageFamily == "darknet") m_ortEngine->setDarknetLayout(true);
|
||||
return true;
|
||||
}
|
||||
if (!FileExist(_modelFilePath)) {
|
||||
|
||||
Reference in New Issue
Block a user