From 0baca1b20de509859bb8fcbf2b133eb69e31c5fc Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Sat, 26 Sep 2026 18:43:13 +1000 Subject: [PATCH] feat(ANSODEngine): run Darknet/YOLO exports ("darknet" family) in ONNXYOLO and RTYOLO A Darknet/YOLO model exported by hank-ai's darknet_onnx_export (v5.1+) has two outputs: confs [B,N,nc], already objectness x class probability, and boxes [B,N,1,4], corners x1,y1,x2,y2 normalized to the input. It trains and predicts on a stretched square and has no NMS in the graph. ONNXYOLO took the 4-D boxes tensor for segmentation proto masks and letterboxed the input, so such a model decoded as garbage; a sealed package named as YOLOV4 went to YOLOOD, which cannot open an .ansmodel at all. - ONNXYOLO / RTYOLO: recognise the layout once at load (package family "darknet", output names confs/boxes, or their shapes), before the seg check; stretch-resize the input (single, batch, and RTYOLO's NV12 path); new postprocessDarknet: best class per candidate, threshold, stretch map to the original image, clip, class-aware NMS. - dllmain: a sealed package passed as modelType 1 (YOLOV4) is routed as detection (3), i.e. to RTYOLO on NVIDIA and ONNXYOLO elsewhere, in CreateANSODHandle, CreateANSODHandleEx and OptimizeModelStr. - Every other model is unchanged: all new branches are gated on the layout. Verified on macOS only: ANSONNXYOLO.cpp passes a syntax check, and the new decode functions, extracted into a standalone harness running the model through onnxruntime, reproduce Darknet's own detections on dog.jpg. NOT yet built on Windows, and the TensorRT path has not run. Co-Authored-By: Claude Opus 5.5 --- modules/ANSODEngine/ANSEngineCommon.h | 2 +- modules/ANSODEngine/ANSONNXYOLO.cpp | 181 +++++++++++++++++++++++++- modules/ANSODEngine/ANSONNXYOLO.h | 22 ++++ modules/ANSODEngine/ANSRTYOLO.cpp | 151 ++++++++++++++++++++- modules/ANSODEngine/ANSRTYOLO.h | 13 ++ modules/ANSODEngine/dllmain.cpp | 17 +++ 6 files changed, 378 insertions(+), 8 deletions(-) diff --git a/modules/ANSODEngine/ANSEngineCommon.h b/modules/ANSODEngine/ANSEngineCommon.h index ff5f58e..422be9e 100644 --- a/modules/ANSODEngine/ANSEngineCommon.h +++ b/modules/ANSODEngine/ANSEngineCommon.h @@ -816,7 +816,7 @@ namespace ANSCENTER bool _fromPackage{ false }; std::vector _packageModel; std::string _packageTask; // detection, classification, … - std::string _packageFamily; // yolo | yolo10 | rfdetr + std::string _packageFamily; // yolo | yolo10 | rfdetr | darknet std::string _packageMarker; // what the zip would have carried std::string _packageName; // what logs call this model int _packageInputWidth{ 0 }; // 0: the package does not say diff --git a/modules/ANSODEngine/ANSONNXYOLO.cpp b/modules/ANSODEngine/ANSONNXYOLO.cpp index 7dabc9a..833b621 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.cpp +++ b/modules/ANSODEngine/ANSONNXYOLO.cpp @@ -24,6 +24,7 @@ namespace ANSCENTER { else { inputImageShape = cv::Size(640, 640); } + detectDarknetLayout(); } ONNXYOLO::ONNXYOLO(const std::string& _onnx_path, EngineType engineType, @@ -39,6 +40,7 @@ namespace ANSCENTER { else { inputImageShape = cv::Size(640, 640); } + detectDarknetLayout(); } ONNXYOLO::ONNXYOLO(const void* modelData, size_t modelSize, @@ -54,6 +56,7 @@ namespace ANSCENTER { else { inputImageShape = cv::Size(640, 640); } + detectDarknetLayout(); } ONNXYOLO::ONNXYOLO(const void* modelData, size_t modelSize, @@ -70,6 +73,44 @@ namespace ANSCENTER { else { inputImageShape = cv::Size(640, 640); } + detectDarknetLayout(); + } + + // ------------------------------------------------------------------ + // detectDarknetLayout — recognise a Darknet/YOLO export (hank-ai + // darknet_onnx_export) from its outputs, once, at load time: + // confs [B,N,nc] (objectness x class probability) + // boxes [B,N,1,4] or [B,N,4] (x1,y1,x2,y2 normalised 0..1) + // Matched by name first, then by shape. Must be decided before the + // segmentation check in detect(), which would otherwise take the 4-D + // boxes tensor for proto masks. + // ------------------------------------------------------------------ + void ONNXYOLO::detectDarknetLayout() + { + isDarknetLayout = false; + if (num_outputs != 2 || output_node_dims.size() != 2) return; + + if (output_node_names_.size() == 2) { + const bool byName = + (output_node_names_[0] == "confs" && output_node_names_[1] == "boxes") || + (output_node_names_[0] == "boxes" && output_node_names_[1] == "confs"); + if (byName) { isDarknetLayout = true; return; } + } + + // By shape: exactly one output ends in 4 (the boxes), and both + // carry the same number of candidates N in dim 1. + auto isBoxes = [](const std::vector& d) { + return (d.size() == 4 && d[2] == 1 && d[3] == 4) || + (d.size() == 3 && d[2] == 4); + }; + const auto& d0 = output_node_dims[0]; + const auto& d1 = output_node_dims[1]; + if (isBoxes(d0) == isBoxes(d1)) return; + const auto& boxes = isBoxes(d0) ? d0 : d1; + const auto& confs = isBoxes(d0) ? d1 : d0; + if (confs.size() != 3 || confs[2] < 1) return; + if (confs[1] <= 0 || confs[1] != boxes[1]) return; + isDarknetLayout = true; } // ------------------------------------------------------------------ @@ -143,6 +184,11 @@ namespace ANSCENTER { // Classification: direct resize (no letterbox padding) — matches ANSONNXCL cv::resize(bgrMat, canvas, cv::Size(inputImageShape.width, inputImageShape.height), 0, 0, cv::INTER_LINEAR); + } else if (isDarknetLayout) { + // Darknet: stretch the whole frame to the network size (no + // letterbox) — Darknet trains and predicts on a stretched square + cv::resize(bgrMat, canvas, cv::Size(inputImageShape.width, inputImageShape.height), + 0, 0, cv::INTER_LINEAR); } else { // Detection/Seg/Pose/OBB: Ultralytics letterbox letterBox(bgrMat, canvas, inputImageShape); @@ -164,10 +210,10 @@ namespace ANSCENTER { if (s_loggedFirst != this) { s_loggedFirst = this; ANS_DBG("ONNXYOLO", - "transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d", + "transformInto[first]: inputImageShape=(%dx%d) canvas=%dx%d cls=%d src=%dx%d dyn=%d darknet=%d", inputImageShape.width, inputImageShape.height, width, height, isClassification ? 1 : 0, mat.cols, mat.rows, - isDynamicInputShape ? 1 : 0); + isDynamicInputShape ? 1 : 0, isDarknetLayout ? 1 : 0); } outDims = { 1, 3, height, width }; @@ -222,6 +268,10 @@ namespace ANSCENTER { // Classification: direct resize (no letterbox) cv::resize(bgrImg, canvas, cv::Size(inputImageShape.width, inputImageShape.height), 0, 0, cv::INTER_LINEAR); + } else if (isDarknetLayout) { + // Darknet: stretch resize (no letterbox) + cv::resize(bgrImg, canvas, cv::Size(inputImageShape.width, inputImageShape.height), + 0, 0, cv::INTER_LINEAR); } else { letterBox(bgrImg, canvas, inputImageShape); } @@ -288,6 +338,7 @@ namespace ANSCENTER { // detect — full pipeline with auto task detection // // Decision logic: + // darknet layout (load time) → darknet detection // 2 outputs (second 4D) → segmentation // 1 output, 2D → classification // 1 output, 3D end2end dim2=6 → detection @@ -366,6 +417,12 @@ namespace ANSCENTER { const size_t numOutputs = outputTensors.size(); + // ── Darknet: confs + boxes (checked before seg — boxes is 4-D) ── + if (isDarknetLayout) { + return postprocessDarknet(image.size(), outputTensors, classNames, + confThreshold, iouThreshold); + } + // ── Segmentation: 2 outputs (detections + proto masks) ────────── if (numOutputs >= 2) { const auto protoShape = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape(); @@ -1392,6 +1449,118 @@ namespace ANSCENTER { return results; } + // ==================================================================== + // DARKNET — postprocess + // + // hank-ai darknet_onnx_export (v5.1+): confs [B,N,nc] is already + // objectness x class probability; boxes [B,N,1,4] holds corners + // x1,y1,x2,y2 normalised 0..1 to the (stretched) network input. The + // graph has no NMS. Best class per candidate, threshold, map to the + // original image (a stretch, so x*origW / y*origH), clip, then + // class-aware NMS. + // ==================================================================== + + std::vector ONNXYOLO::postprocessDarknet( + const cv::Size& originalImageSize, + std::vector& outputTensors, + const std::vector& classNames, + float confThreshold, float iouThreshold, int maxDet) + { + if (outputTensors.size() < 2) return {}; + + // Match by name first, then by shape (the boxes tensor ends in 4) + int confsIdx = -1, boxesIdx = -1; + for (size_t t = 0; t < outputTensors.size() && t < output_node_names_.size(); ++t) { + if (output_node_names_[t] == "confs") confsIdx = static_cast(t); + else if (output_node_names_[t] == "boxes") boxesIdx = static_cast(t); + } + if (confsIdx < 0 || boxesIdx < 0) { + confsIdx = boxesIdx = -1; + for (size_t t = 0; t < outputTensors.size(); ++t) { + const auto sh = outputTensors[t].GetTensorTypeAndShapeInfo().GetShape(); + if (!sh.empty() && sh.back() == 4 && boxesIdx < 0) boxesIdx = static_cast(t); + else if (confsIdx < 0) confsIdx = static_cast(t); + } + } + if (confsIdx < 0 || boxesIdx < 0) return {}; + + const auto confsShape = outputTensors[confsIdx].GetTensorTypeAndShapeInfo().GetShape(); + const auto boxesInfo = outputTensors[boxesIdx].GetTensorTypeAndShapeInfo(); + if (confsShape.size() != 3) return {}; + + const int numBoxes = static_cast(confsShape[1]); + const int numClasses = static_cast(confsShape[2]); + if (numBoxes <= 0 || numClasses <= 0) return {}; + if (boxesInfo.GetElementCount() != static_cast(numBoxes) * 4) return {}; + + const float* confs = outputTensors[confsIdx].GetTensorMutableData(); + const float* boxes = outputTensors[boxesIdx].GetTensorMutableData(); + + const float origW = static_cast(originalImageSize.width); + const float origH = static_cast(originalImageSize.height); + + struct Candidate { float x1, y1, x2, y2, conf; int classId; }; + std::vector candidates; + + for (int i = 0; i < numBoxes; ++i) { + const float* scoresPtr = confs + static_cast(i) * numClasses; + float maxScore = scoresPtr[0]; + int bestClass = 0; + for (int c = 1; c < numClasses; ++c) { + if (scoresPtr[c] > maxScore) { maxScore = scoresPtr[c]; bestClass = c; } + } + if (maxScore <= confThreshold) continue; + + const float* b = boxes + static_cast(i) * 4; + float x1 = clamp(b[0] * origW, 0.f, origW); + float y1 = clamp(b[1] * origH, 0.f, origH); + float x2 = clamp(b[2] * origW, 0.f, origW); + float y2 = clamp(b[3] * origH, 0.f, origH); + if (x2 - x1 < 1.f || y2 - y1 < 1.f) continue; + candidates.push_back({ x1, y1, x2, y2, maxScore, bestClass }); + } + if (candidates.empty()) return {}; + + // Class-aware NMS (boxes are already in image pixels, so compare + // class ids rather than offsetting by a fixed MAX_WH) + std::vector sortedIdx(candidates.size()); + std::iota(sortedIdx.begin(), sortedIdx.end(), 0); + std::sort(sortedIdx.begin(), sortedIdx.end(), + [&](int a, int b) { return candidates[a].conf > candidates[b].conf; }); + + std::vector suppressed(sortedIdx.size(), false); + std::vector results; + results.reserve(std::min(static_cast(sortedIdx.size()), maxDet)); + + for (size_t i = 0; i < sortedIdx.size() && static_cast(results.size()) < maxDet; ++i) { + if (suppressed[i]) continue; + const auto& cur = candidates[sortedIdx[i]]; + const float curArea = (cur.x2 - cur.x1) * (cur.y2 - cur.y1); + + for (size_t j = i + 1; j < sortedIdx.size(); ++j) { + if (suppressed[j]) continue; + const auto& o = candidates[sortedIdx[j]]; + if (o.classId != cur.classId) continue; + float iw = std::min(cur.x2, o.x2) - std::max(cur.x1, o.x1); + float ih = std::min(cur.y2, o.y2) - std::max(cur.y1, o.y1); + if (iw <= 0.f || ih <= 0.f) continue; + float inter = iw * ih; + float ua = curArea + (o.x2 - o.x1) * (o.y2 - o.y1) - inter; + if (ua > 0.f && inter / ua > iouThreshold) suppressed[j] = true; + } + + Object obj; + obj.classId = cur.classId; + obj.confidence = cur.conf; + obj.box = cv::Rect(static_cast(cur.x1), static_cast(cur.y1), + static_cast(cur.x2 - cur.x1), static_cast(cur.y2 - cur.y1)); + if (cur.classId >= 0 && cur.classId < static_cast(classNames.size())) + obj.className = classNames[cur.classId]; + results.push_back(std::move(obj)); + } + return results; + } + // ==================================================================== // CLASSIFICATION — postprocess // ==================================================================== @@ -1569,6 +1738,11 @@ namespace ANSCENTER { } // Dispatch to correct postprocess method + if (isDarknetLayout) { + results[i] = postprocessDarknet(originalSizes[i], perImageOutputs, classNames, + confThreshold, iouThreshold); + continue; + } if (numOutputs >= 2) { const auto protoShape = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape(); if (protoShape.size() == 4) { @@ -1718,6 +1892,8 @@ namespace ANSCENTER { } m_ortEngine = std::make_shared( _packageModel.data(), _packageModel.size(), _packageName); + // The manifest's family outranks what the outputs look like. + if (_packageFamily == "darknet") m_ortEngine->setDarknetLayout(true); return true; } if (!FileExist(_modelFilePath)) { @@ -1744,6 +1920,7 @@ namespace ANSCENTER { } m_ortEngine = std::make_shared( _packageModel.data(), _packageModel.size(), _packageName, engineType); + if (_packageFamily == "darknet") m_ortEngine->setDarknetLayout(true); return true; } if (!FileExist(_modelFilePath)) { diff --git a/modules/ANSODEngine/ANSONNXYOLO.h b/modules/ANSODEngine/ANSONNXYOLO.h index 80f6e9a..3664f63 100644 --- a/modules/ANSODEngine/ANSONNXYOLO.h +++ b/modules/ANSODEngine/ANSONNXYOLO.h @@ -19,6 +19,7 @@ namespace ANSCENTER { // - Segmentation: [B,300,38]+protos or [B, nc+36, N]+protos // - Pose : [B,300,6+nk*3] or [B, nc+4+nk*3, N] legacy // - Classification: [B, nc] + // - Darknet detect: confs [B,N,nc] + boxes [B,N,1,4] (stretch resize) // // Preprocessing follows the exact Ultralytics LetterBox transform: // - center=True, stride=32, pad_value=114 @@ -95,6 +96,14 @@ namespace ANSCENTER { /// True if the ONNX model has dynamic spatial dimensions. bool hasDynamicInputShape() const { return isDynamicInputShape; } + /// Darknet/YOLO layout (hank-ai darknet_onnx_export): outputs + /// confs [B,N,nc] + boxes [B,N,1,4], stretch resize, no letterbox. + /// Detected from the output names/shapes at construction; the + /// wrapper also forces it when an .ansmodel package says its + /// family is "darknet". + void setDarknetLayout(bool darknet) { isDarknetLayout = darknet; } + bool hasDarknetLayout() const { return isDarknetLayout; } + private: Ort::Value transform(const cv::Mat& mat) override; Ort::Value transformBatch(const std::vector& images) override; @@ -155,6 +164,13 @@ namespace ANSCENTER { const std::vector& classNames, float confThreshold, float iouThreshold, int numKPS, int maxDet = 300); + // ── Darknet postprocess (confs + boxes, no NMS in the graph) ──── + std::vector postprocessDarknet( + const cv::Size& originalImageSize, + std::vector& outputTensors, + const std::vector& classNames, + float confThreshold, float iouThreshold, int maxDet = 300); + // ── Classification postprocess ────────────────────────────────── std::vector postprocessClassify( std::vector& outputTensors, @@ -189,6 +205,12 @@ namespace ANSCENTER { cv::Size inputImageShape; bool isDynamicInputShape{ false }; + // Darknet layout: decided once at load time (see setDarknetLayout). + // Selects the stretch resize in transformInto / transformBatchInto + // and postprocessDarknet in detect / detectBatch. + bool isDarknetLayout{ false }; + void detectDarknetLayout(); + // Per-call preprocess that writes into caller-provided buffer + dims // instead of the BasicOrtHandler members (input_values_handler / // input_node_dims). detect() / detectBatch() use these so multiple diff --git a/modules/ANSODEngine/ANSRTYOLO.cpp b/modules/ANSODEngine/ANSRTYOLO.cpp index fcba79e..093a030 100644 --- a/modules/ANSODEngine/ANSRTYOLO.cpp +++ b/modules/ANSODEngine/ANSRTYOLO.cpp @@ -169,6 +169,7 @@ namespace ANSCENTER { } m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize; m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize; + DetectDarknetLayout(); m_trtEngine->warmUp(); } _modelLoadValid = true; @@ -273,6 +274,7 @@ namespace ANSCENTER { } m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize; m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize; + DetectDarknetLayout(); m_trtEngine->warmUp(); } _modelLoadValid = true; @@ -388,6 +390,7 @@ namespace ANSCENTER { } m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize; m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize; + DetectDarknetLayout(); m_trtEngine->warmUp(); } _modelLoadValid = true; @@ -511,6 +514,9 @@ namespace ANSCENTER { if (srcImg.rows != inputH || srcImg.cols != inputW) { if (isClassification) { cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR); + } else if (_isDarknet) { + // Darknet: stretch the whole frame (no letterbox) + cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR); } else { cpuResized = Engine::cpuResizeKeepAspectRatioPadRightBottom(srcImg, inputH, inputW); } @@ -921,12 +927,15 @@ namespace ANSCENTER { const float scaleW = inputW / static_cast(srcImg.cols); const float scaleH = inputH / static_cast(srcImg.rows); - outMetadata.ratios[i] = isClassification ? 1.f : 1.f / std::min(scaleW, scaleH); + outMetadata.ratios[i] = (isClassification || _isDarknet) ? 1.f : 1.f / std::min(scaleW, scaleH); cv::Mat cpuResized; if (srcImg.rows != inputH || srcImg.cols != inputW) { if (isClassification) { cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR); + } else if (_isDarknet) { + // Darknet: stretch the whole frame (no letterbox) + cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR); } else { cpuResized = Engine::cpuResizeKeepAspectRatioPadRightBottom(srcImg, inputH, inputW); } @@ -1814,6 +1823,117 @@ namespace ANSCENTER { } } + // ==================================================================== + // Darknet — layout detection + postprocess + // + // hank-ai darknet_onnx_export (v5.1+): confs [B,N,nc] is already + // objectness x class probability; boxes [B,N,1,4] holds corners + // x1,y1,x2,y2 normalised 0..1 to the (stretched) network input. The + // graph has no NMS. Best class per candidate, threshold, map to the + // original image (a stretch, so x*imgWidth / y*imgHeight), clip, then + // class-aware NMS. TensorRT exposes no output names here, so the + // boxes tensor is the one whose last dim is 4. + // ==================================================================== + void ANSRTYOLO::DetectDarknetLayout() { + _isDarknet = false; + if (_fromPackage && _packageFamily == "darknet") { + _isDarknet = true; + return; + } + if (!m_trtEngine) return; + const auto& outputDims = m_trtEngine->getOutputDims(); + if (outputDims.size() != 2) return; + + auto isBoxes = [](const nvinfer1::Dims& d) { + return (d.nbDims == 4 && d.d[2] == 1 && d.d[3] == 4) || + (d.nbDims == 3 && d.d[2] == 4); + }; + const bool b0 = isBoxes(outputDims[0]); + const bool b1 = isBoxes(outputDims[1]); + if (b0 == b1) return; + const auto& boxes = b0 ? outputDims[0] : outputDims[1]; + const auto& confs = b0 ? outputDims[1] : outputDims[0]; + if (confs.nbDims != 3 || confs.d[2] < 1) return; + if (confs.d[1] <= 0 || confs.d[1] != boxes.d[1]) return; + _isDarknet = true; + } + + std::vector ANSRTYOLO::PostprocessDarknet( + std::vector>& featureVectors, + const std::string& camera_id, const ImageMetadata& meta) { + try { + const auto& outputDims = m_trtEngine->getOutputDims(); + if (outputDims.size() < 2 || featureVectors.size() < 2) return {}; + + int boxesIdx = -1, confsIdx = -1; + for (size_t t = 0; t < outputDims.size(); ++t) { + const auto& d = outputDims[t]; + if (d.nbDims > 0 && d.d[d.nbDims - 1] == 4 && boxesIdx < 0) boxesIdx = static_cast(t); + else if (confsIdx < 0) confsIdx = static_cast(t); + } + if (boxesIdx < 0 || confsIdx < 0) return {}; + if (outputDims[confsIdx].nbDims != 3) return {}; + + const int numClasses = outputDims[confsIdx].d[2]; + if (numClasses <= 0) return {}; + const auto& confsVec = featureVectors[confsIdx]; + const auto& boxesVec = featureVectors[boxesIdx]; + // Number of candidates from the data (dim 1 may be dynamic) + const int numBoxes = static_cast(confsVec.size() / numClasses); + if (numBoxes <= 0 || boxesVec.size() < static_cast(numBoxes) * 4) return {}; + + std::vector bboxes; + std::vector scores; + std::vector labels; + std::vector indices; + + for (int i = 0; i < numBoxes; i++) { + const float* scoresPtr = confsVec.data() + static_cast(i) * numClasses; + const float* maxSPtr = std::max_element(scoresPtr, scoresPtr + numClasses); + float score = *maxSPtr; + if (score > _modelConfig.detectionScoreThreshold) { + const float* b = boxesVec.data() + static_cast(i) * 4; + float x0 = std::clamp(b[0] * meta.imgWidth, 0.f, meta.imgWidth); + float y0 = std::clamp(b[1] * meta.imgHeight, 0.f, meta.imgHeight); + float x1 = std::clamp(b[2] * meta.imgWidth, 0.f, meta.imgWidth); + float y1 = std::clamp(b[3] * meta.imgHeight, 0.f, meta.imgHeight); + if (x1 - x0 < 1.f || y1 - y0 < 1.f) continue; + cv::Rect_ bbox; + bbox.x = x0; bbox.y = y0; + bbox.width = x1 - x0; bbox.height = y1 - y0; + bboxes.push_back(bbox); + labels.push_back(static_cast(maxSPtr - scoresPtr)); + scores.push_back(score); + } + } + + cv::dnn::NMSBoxesBatched(bboxes, scores, labels, PROBABILITY_THRESHOLD, NMS_THRESHOLD, indices); + std::vector objects; + int classNameSize = static_cast(_classes.size()); + for (auto& chosenIdx : indices) { + if (static_cast(objects.size()) >= TOP_K) break; + if (scores[chosenIdx] > _modelConfig.detectionScoreThreshold) { + Object obj{}; + obj.confidence = scores[chosenIdx]; + obj.classId = labels[chosenIdx]; + obj.box = bboxes[chosenIdx]; + if (!_classes.empty()) { + obj.className = (obj.classId < classNameSize) + ? _classes[obj.classId] : _classes[classNameSize - 1]; + } + else { obj.className = "Unknown"; } + obj.cameraId = camera_id; + objects.push_back(obj); + } + } + return objects; + } + catch (std::exception& e) { + _logger.LogFatal("ANSRTYOLO::PostprocessDarknet", e.what(), __FILE__, __LINE__); + return {}; + } + } + // ==================================================================== // DetectObjects — single image with auto-detection of task type // ==================================================================== @@ -1856,8 +1976,11 @@ namespace ANSCENTER { const int inputW = inputDims[0].d[2]; const int inputH = inputDims[0].d[1]; + // Darknet stretches rather than letterboxes: the direct + // (no-padding) resize kernel is the classification one. auto nv12 = m_nv12Helper.tryNV12(inputImage, inferenceGpu, inputW, inputH, - NV12PreprocessHelper::defaultYOLOLauncher(), + _isDarknet ? NV12PreprocessHelper::classificationLauncher() + : NV12PreprocessHelper::defaultYOLOLauncher(), _logger, "ANSRTYOLO"); if (nv12.succeeded) { meta.imgWidth = nv12.metaWidth; @@ -1912,7 +2035,14 @@ namespace ANSCENTER { const auto& outputDims = m_trtEngine->getOutputDims(); const size_t numOutputs = outputDims.size(); - if (numOutputs >= 2) { + // Darknet (confs + boxes) — checked before seg, whose test + // (2 outputs) it would otherwise pass + if (_isDarknet) { + std::vector> featureVector2d; + Engine::transformOutput(featureVectors, featureVector2d); + results = PostprocessDarknet(featureVector2d, camera_id, meta); + } + else if (numOutputs >= 2) { std::vector> featureVector2d; Engine::transformOutput(featureVectors, featureVector2d); double msTransform = dbg ? elapsed() : 0; @@ -2212,13 +2342,16 @@ namespace ANSCENTER { int nc = static_cast(_classes.size()); enum class TaskType { DetLegacy, DetE2E, OBBLegacy, OBBE2E, - SegLegacy, SegE2E, PoseLegacy, PoseE2E, Classify }; + SegLegacy, SegE2E, PoseLegacy, PoseE2E, Classify, Darknet }; TaskType taskType = TaskType::DetLegacy; // default // E2E: dim1 > dim2 (e.g. [B,300,6]); Legacy: dim1 < dim2 (e.g. [B,84,8400]) const bool isEndToEnd = (dim1 > dim2) || (dim2 <= 20); - if (numOutputs >= 2) { + if (_isDarknet) { + taskType = TaskType::Darknet; + } + else if (numOutputs >= 2) { taskType = isEndToEnd ? TaskType::SegE2E : TaskType::SegLegacy; } else if (outputDims[0].nbDims <= 2) { @@ -2299,6 +2432,14 @@ namespace ANSCENTER { } break; } + case TaskType::Darknet: { + std::vector> fv2d(batchOutput.begin(), batchOutput.end()); + postFutures.push_back(std::async(std::launch::async, + [this, fv = std::move(fv2d), cid = camera_id, m = imgMeta]() mutable { + return PostprocessDarknet(fv, cid, m); + })); + break; + } case TaskType::Classify: { std::vector fv = batchOutput.empty() ? std::vector{} : batchOutput[0]; postFutures.push_back(std::async(std::launch::async, diff --git a/modules/ANSODEngine/ANSRTYOLO.h b/modules/ANSODEngine/ANSRTYOLO.h index 5fd7b5b..68f5252 100644 --- a/modules/ANSODEngine/ANSRTYOLO.h +++ b/modules/ANSODEngine/ANSRTYOLO.h @@ -22,6 +22,7 @@ namespace ANSCENTER { // - Segmentation: legacy [B, nc+36, N]+protos or end2end [B, 300, 38]+protos // - Pose : legacy [B, nc+4+nk*3, N] or end2end [B, 300, 6+nk*3] // - Classification: [B, nc] + // - Darknet detect: confs [B,N,nc] + boxes [B,N,1,4] (stretch resize) // // Uses Engine with GPU preprocessing (cv::cuda::GpuMat), // supports multi-GPU pool via SetMaxSlotsPerGpu. @@ -124,6 +125,18 @@ namespace ANSCENTER { std::vector& featureVector, const std::string& camera_id, const ImageMetadata& meta); + // ── Darknet postprocessor (confs + boxes, no NMS in the graph) ─── + std::vector PostprocessDarknet( + std::vector>& featureVectors, + const std::string& camera_id, const ImageMetadata& meta); + + // Darknet/YOLO layout (hank-ai darknet_onnx_export): package family + // "darknet", or outputs shaped confs [B,N,nc] + boxes [B,N,1,4]. + // Decided once, after the engine is loaded; selects the stretch + // resize (no letterbox) and PostprocessDarknet. + bool _isDarknet{ false }; + void DetectDarknetLayout(); + // ── OBB NMS helpers (Prob-IoU based) ───────────────────────────── struct OrientedBox { float x, y, width, height, angle; diff --git a/modules/ANSODEngine/dllmain.cpp b/modules/ANSODEngine/dllmain.cpp index 609e3e6..d06e971 100644 --- a/modules/ANSODEngine/dllmain.cpp +++ b/modules/ANSODEngine/dllmain.cpp @@ -434,6 +434,13 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase** // handler. Unlisted modelType values are left untouched for the switch below. // See CreateANSODHandleEx for the full rationale — three correctness bugs // were fixed in that dispatcher and must be kept in sync across copies. + // A sealed package is an ONNX model whatever legacy type the caller names. + // YOLOV4 = 1 goes to YOLOOD, which opens only a zip of .cfg/.weights and + // cannot read an .ansmodel; a Darknet/YOLO package (family "darknet") runs + // in the YOLO engines, which recognise its confs/boxes head. Routed as + // detection so the switch below picks TensorRT or ONNX for the hardware. + if (modelType == 1 && ANSCENTER::modelpkg::IsPackagePath(modelFilePath ? modelFilePath : "")) + modelType = 3; const bool onNvidia = (engineType == ANSCENTER::EngineType::NVIDIA_GPU); switch (modelType) { // ── Detection family: YOLOv8 / V11 / V12 / generic TRT / V10-RTOD ── @@ -816,6 +823,13 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl // passing modelType=20 for classification ended up with a YOLO head. // (3) modelType == 22 / 24 (TRT pose / TRT seg) on a non-NVIDIA box fell // back to ONNXYOLO instead of the correct ONNXPOSE / ONNXSEG handler. + // A sealed package is an ONNX model whatever legacy type the caller names. + // YOLOV4 = 1 goes to YOLOOD, which opens only a zip of .cfg/.weights and + // cannot read an .ansmodel; a Darknet/YOLO package (family "darknet") runs + // in the YOLO engines, which recognise its confs/boxes head. Routed as + // detection so the switch below picks TensorRT or ONNX for the hardware. + if (modelType == 1 && ANSCENTER::modelpkg::IsPackagePath(modelFilePath ? modelFilePath : "")) + modelType = 3; const bool onNvidia = (engineType == ANSCENTER::EngineType::NVIDIA_GPU); switch (modelType) { // ── Detection family: YOLOv8 / V11 / V12 / generic TRT / V10-RTOD ── @@ -1589,6 +1603,9 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode // inference tasks. The global mutex was blocking ALL running tasks' // result delivery for the entire duration of TRT engine building. ANSCENTER::EngineType engineType = ANSCENTER::ANSLicenseHelper::CheckHardwareInformation(); + // Same rule as CreateANSODHandleEx: a sealed Darknet/YOLO package named + // as YOLOV4 = 1 is built in the YOLO engines, not YOLOOD. + if (modelType == 1 && sealedPackage) modelType = 3; // Route detection / pose / segmentation / OBB / classification to the best